<s> """<STR_LIT>""" <EOL> import sys <EOL> import tempfile <EOL> import optparse <EOL> import shutil <EOL> import itertools <EOL> import csv <EOL> import math <EOL> import random <EOL> import re <EOL> import glob <EOL> import os <EOL> import shutil <EOL> import collections <EOL> import CGAT . Experiment as E <EOL> import CGAT . Pipeline as P <EOL> from ruffus import * <EOL> import csv <EOL> import sqlite3 <EOL> import CGAT . IndexedFasta as IndexedFasta <EOL> import CGAT . IndexedGenome as IndexedGenome <EOL> import CGAT . FastaIterator as FastaIterator <EOL> import CGAT . Genomics as Genomics <EOL> import CGAT . IOTools as IOTools <EOL> import CGAT . MAST as MAST <EOL> import CGAT . GTF as GTF <EOL> import CGAT . GFF as GFF <EOL> import CGAT . Bed as Bed <EOL> import CGAT . Stats as Stats <EOL> import cStringIO <EOL> import pysam <EOL> import numpy <EOL> import gzip <EOL> import CGAT . Expression as Expression <EOL> import CGAT . Masker as Masker <EOL> import CGAT . Glam2Scan as Glam2Scan <EOL> import fileinput <EOL> import CGAT . Motifs as Motifs <EOL> try : import nubiscan <EOL> except ImportError : pass <EOL> import gff2annotator <EOL> import CGAT . Bioprospector as Bioprospector <EOL> import pipeline_vitaminD_annotator as PAnnotator <EOL> import pipeline_vitaminD_expression as PExpression <EOL> import PipelineChipseq as PipelineChipseq <EOL> import PipelineMotifs as PipelineMotifs <EOL> import PipelineGeneset as PGeneset <EOL> import time <EOL> if os . path . exists ( "<STR_LIT>" ) : <EOL> execfile ( "<STR_LIT>" ) <EOL> TARGET_ANNOTATION = '<STR_LIT>' <EOL> TARGET_GENESET = '<STR_LIT>' <EOL> TARGET_PROMOTORS = '<STR_LIT>' <EOL> TARGET_TSS = '<STR_LIT>' <EOL> TARGET_REPEATS = '<STR_LIT>' <EOL> TARGET_TRANSCRIPTS = '<STR_LIT>' <EOL> TARGET_PROBESET = '<STR_LIT>' <EOL> TARGET_TRANSCRIPTS_TSS = '<STR_LIT>' <EOL> TARGET_TRANSCRIPTS_PROMOTORS = '<STR_LIT>' <EOL> TARGET_ANNOTATOR_GENETERRITORIES = '<STR_LIT>' <EOL> TARGET_MAPPABILITY = '<STR_LIT>' <EOL> BAM_SUFFIX = "<STR_LIT>" <EOL> PARAMS = P . getParameters ( ) <EOL> @ files ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> def indexGenome ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> pysam . faidx ( outfile ) <EOL> def getUCSCTracks ( infile = PARAMS [ "<STR_LIT>" ] ) : <EOL> '''<STR_LIT>''' <EOL> tables = [ ] <EOL> with open ( infile ) as f : <EOL> for line in f : <EOL> if line . startswith ( "<STR_LIT:#>" ) : continue <EOL> tablename = line [ : - <NUM_LIT:1> ] . strip ( ) <EOL> if tablename == "<STR_LIT>" : continue <EOL> tables . append ( tablename ) <EOL> return tables <EOL> @ files ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> def buildGenomeGCSegmentation ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> to_cluster = True <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ files ( buildGenomeGCSegmentation , "<STR_LIT>" ) <EOL> def buildAnnotatorGC ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> to_cluster = True <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> @ posttask ( touch_file ( "<STR_LIT>" ) ) <EOL> @ files ( PARAMS [ "<STR_LIT>" ] , "<STR_LIT>" ) <EOL> def importUCSCEncodeTracks ( infile , outfile ) : <EOL> statement = '''<STR_LIT>''' <EOL> dbhandle = sqlite3 . connect ( PARAMS [ "<STR_LIT>" ] ) <EOL> cc = dbhandle . cursor ( ) <EOL> tables = set ( [ x [ <NUM_LIT:0> ] for x in cc . execute ( "<STR_LIT>" ) ] ) <EOL> cc . close ( ) <EOL> for tablename in getUCSCTracks ( infile ) : <EOL> if tablename in tables : <EOL> E . info ( "<STR_LIT>" % locals ( ) ) <EOL> continue <EOL> E . info ( "<STR_LIT>" % locals ( ) ) <EOL> P . run ( ) <EOL> @ files ( PARAMS [ "<STR_LIT>" ] , PARAMS [ "<STR_LIT>" ] ) <EOL> def exportUCSCMappabilityTrackToBed ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> infile = gzip . open ( infile , "<STR_LIT:r>" ) <EOL> outf = open ( outfile , "<STR_LIT:w>" ) <EOL> for line in infile : <EOL> if line . startswith ( "<STR_LIT>" ) : <EOL> contig , start , step = re . match ( "<STR_LIT>" , line ) . groups ( ) <EOL> start , step = int ( start ) - <NUM_LIT:1> , int ( step ) <EOL> end = start + step <EOL> last_val = None <EOL> else : <EOL> val = int ( line ) <EOL> if last_val != val : <EOL> if last_val == <NUM_LIT:1> : <EOL> outf . write ( "<STR_LIT:\t>" . join ( ( contig , str ( start ) , str ( end ) ) ) + "<STR_LIT:\n>" ) <EOL> start = end <EOL> end = start + step <EOL> else : <EOL> end += step <EOL> last_val = val <EOL> outf . close ( ) <EOL> @ transform ( importUCSCEncodeTracks , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def exportUCSCEncodeTracks ( infile , outfile ) : <EOL> dbhandle = sqlite3 . connect ( PARAMS [ "<STR_LIT>" ] ) <EOL> outs = open ( outfile , "<STR_LIT:w>" ) <EOL> for tablename in getUCSCTracks ( ) : <EOL> outs . write ( "<STR_LIT>" % tablename ) <EOL> cc = dbhandle . cursor ( ) <EOL> statement = "<STR_LIT>" % ( tablename ) <EOL> cc . execute ( statement ) <EOL> for contig , start , end in cc : <EOL> outs . write ( "<STR_LIT>" % ( contig , start , end ) ) <EOL> outs . close ( ) <EOL> @ files ( PARAMS [ "<STR_LIT>" ] , "<STR_LIT>" ) <EOL> def importTranscriptInformation ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PGeneset . importTranscriptInformation ( infile , outfile ) <EOL> @ files ( PARAMS [ "<STR_LIT>" ] , "<STR_LIT>" ) <EOL> def importGeneInformation ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PGeneset . importGeneInformation ( infile , outfile ) <EOL> @ files ( ( ( PARAMS [ "<STR_LIT>" ] , "<STR_LIT>" ) , <EOL> ( PARAMS [ "<STR_LIT>" ] , "<STR_LIT>" ) ) ) <EOL> def importRegionsOfInterest ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> table = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ merge ( PARAMS [ "<STR_LIT>" ] , "<STR_LIT>" ) <EOL> def importSNPs ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> reader = csv . DictReader ( open ( infile , "<STR_LIT>" ) ) <EOL> ids = { } <EOL> for row in reader : <EOL> roi_id , contig , start , end , snps = row [ "<STR_LIT>" ] , row [ "<STR_LIT>" ] , row [ "<STR_LIT:start>" ] , row [ "<STR_LIT:end>" ] , row [ "<STR_LIT>" ] <EOL> snps = snps . split ( "<STR_LIT:U+002C>" ) <EOL> for snp in snps : <EOL> s = snp . strip ( ) <EOL> if s == "<STR_LIT>" or s == "<STR_LIT>" : continue <EOL> try : <EOL> ids [ snp . strip ( ) ] = ( roi_id , contig , int ( start ) , int ( end ) ) <EOL> except ValueError : <EOL> continue <EOL> tmpf = P . getTempFile ( ) <EOL> tmpf . write ( "<STR_LIT>" ) <EOL> counter = E . Counter ( ) <EOL> found = set ( ) <EOL> inf = gzip . open ( PARAMS [ "<STR_LIT>" ] , "<STR_LIT:r>" ) <EOL> for line in inf : <EOL> data = line [ : - <NUM_LIT:1> ] . split ( "<STR_LIT:\t>" ) <EOL> contig , snp , pos = data [ <NUM_LIT:1> ] , data [ <NUM_LIT:4> ] , data [ <NUM_LIT:2> ] <EOL> if snp not in ids : continue <EOL> roi_id , roi_contig , roi_start , roi_end = ids [ snp ] <EOL> if contig != roi_contig or not roi_start <= int ( pos ) < roi_end : <EOL> counter . out_of_range += <NUM_LIT:1> <EOL> continue <EOL> counter . accepted += <NUM_LIT:1> <EOL> tmpf . write ( "<STR_LIT>" % ( roi_id , snp , contig , pos ) ) <EOL> found . add ( snp ) <EOL> inf . close ( ) <EOL> table = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> tmpfilename = tmpf . name <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> os . unlink ( tmpfilename ) <EOL> outfile = open ( outfile , "<STR_LIT:a>" ) <EOL> outfile . write ( "<STR_LIT>" % ( len ( ids ) , len ( found ) , str ( counter ) ) ) <EOL> outfile . write ( "<STR_LIT>" % "<STR_LIT:U+002C>" . join ( set ( ids . keys ( ) ) . difference ( found ) ) ) <EOL> outfile . close ( ) <EOL> E . info ( "<STR_LIT>" % ( len ( ids ) , len ( found ) , str ( counter ) ) ) <EOL> if len ( ids ) != len ( found ) : <EOL> E . warn ( "<STR_LIT>" % "<STR_LIT:U+002C>" . join ( set ( ids . keys ( ) ) . difference ( found ) ) ) <EOL> @ merge ( PARAMS [ "<STR_LIT>" ] , "<STR_LIT>" ) <EOL> def importGWAS ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> reader = csv . DictReader ( open ( infile , "<STR_LIT>" ) ) <EOL> ids = collections . defaultdict ( list ) <EOL> last_gwas = None <EOL> for row in reader : <EOL> snps , gwas , genes = row [ "<STR_LIT>" ] , row [ "<STR_LIT:class>" ] , row [ "<STR_LIT>" ] <EOL> if gwas == "<STR_LIT>" : gwas = last_gwas <EOL> snps = snps . split ( "<STR_LIT:U+002C>" ) <EOL> for snp in snps : <EOL> s = snp . strip ( ) <EOL> if s == "<STR_LIT>" or s == "<STR_LIT>" : continue <EOL> genes = re . sub ( "<STR_LIT:U+0020>" , "<STR_LIT>" , genes . lower ( ) ) <EOL> ids [ s . strip ( ) ] . append ( ( gwas , genes ) ) <EOL> last_gwas = gwas <EOL> tmpf = P . getTempFile ( ) <EOL> tmpf . write ( "<STR_LIT>" % "<STR_LIT:\t>" . join ( ( "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:class>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:start>" , "<STR_LIT:end>" , "<STR_LIT>" ) ) ) <EOL> counter = E . Counter ( ) <EOL> found = set ( ) <EOL> inf = gzip . open ( PARAMS [ "<STR_LIT>" ] , "<STR_LIT:r>" ) <EOL> nsnps = <NUM_LIT:0> <EOL> halfwidth = PARAMS [ "<STR_LIT>" ] <EOL> for line in inf : <EOL> data = line [ : - <NUM_LIT:1> ] . split ( "<STR_LIT:\t>" ) <EOL> contig , snp , pos = data [ <NUM_LIT:1> ] , data [ <NUM_LIT:4> ] , data [ <NUM_LIT:2> ] <EOL> if snp not in ids : continue <EOL> if contig == None or contig == "<STR_LIT>" : <EOL> counter . unmapped += <NUM_LIT:1> <EOL> continue <EOL> counter . accepted += <NUM_LIT:1> <EOL> pos = int ( pos ) <EOL> for gwas , genes in ids [ snp ] : <EOL> nsnps += <NUM_LIT:1> <EOL> tmpf . write ( "<STR_LIT>" % "<STR_LIT:\t>" . join ( map ( str , <EOL> ( nsnps , <EOL> snp , <EOL> gwas , <EOL> contig , <EOL> pos , <EOL> pos - halfwidth , <EOL> pos + halfwidth , <EOL> genes <EOL> ) ) ) ) <EOL> found . add ( snp ) <EOL> inf . close ( ) <EOL> tmpf . close ( ) <EOL> table = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> tmpfilename = tmpf . name <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** locals ( ) ) <EOL> os . unlink ( tmpfilename ) <EOL> outfile = open ( outfile , "<STR_LIT:a>" ) <EOL> outfile . write ( "<STR_LIT>" % ( len ( ids ) , len ( found ) , str ( counter ) ) ) <EOL> outfile . write ( "<STR_LIT>" % "<STR_LIT:U+002C>" . join ( set ( ids . keys ( ) ) . difference ( found ) ) ) <EOL> outfile . close ( ) <EOL> E . info ( "<STR_LIT>" % ( len ( ids ) , len ( found ) , str ( counter ) ) ) <EOL> if len ( ids ) != len ( found ) : <EOL> E . warn ( "<STR_LIT>" % "<STR_LIT:U+002C>" . join ( set ( ids . keys ( ) ) . difference ( found ) ) ) <EOL> @ merge ( PARAMS [ "<STR_LIT>" ] , "<STR_LIT>" ) <EOL> def importMergedGWAS ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> reader = csv . DictReader ( open ( infile , "<STR_LIT>" ) ) <EOL> ids = collections . defaultdict ( list ) <EOL> last_gwas = None <EOL> for row in reader : <EOL> snps , gwas , genes = row [ "<STR_LIT>" ] , row [ "<STR_LIT:class>" ] , row [ "<STR_LIT>" ] <EOL> if gwas == "<STR_LIT>" : gwas = last_gwas <EOL> snps = snps . split ( "<STR_LIT:U+002C>" ) <EOL> for snp in snps : <EOL> s = snp . strip ( ) <EOL> if s == "<STR_LIT>" or s == "<STR_LIT>" : continue <EOL> genes = re . sub ( "<STR_LIT:U+0020>" , "<STR_LIT>" , genes . lower ( ) ) <EOL> ids [ s . strip ( ) ] . append ( ( gwas , genes ) ) <EOL> last_gwas = gwas <EOL> counter = E . Counter ( ) <EOL> found = set ( ) <EOL> inf = gzip . open ( PARAMS [ "<STR_LIT>" ] , "<STR_LIT:r>" ) <EOL> halfwidth = PARAMS [ "<STR_LIT>" ] <EOL> intervals = collections . defaultdict ( list ) <EOL> for line in inf : <EOL> data = line [ : - <NUM_LIT:1> ] . split ( "<STR_LIT:\t>" ) <EOL> contig , snp , pos = data [ <NUM_LIT:1> ] , data [ <NUM_LIT:4> ] , data [ <NUM_LIT:2> ] <EOL> if snp not in ids : continue <EOL> if contig == None or contig == "<STR_LIT>" : <EOL> counter . unmapped += <NUM_LIT:1> <EOL> continue <EOL> counter . accepted += <NUM_LIT:1> <EOL> pos = int ( pos ) <EOL> for gwas , genes in ids [ snp ] : <EOL> intervals [ gwas ] . append ( ( contig , pos , snp ) ) <EOL> found . add ( snp ) <EOL> inf . close ( ) <EOL> tmpf = P . getTempFile ( ) <EOL> tmpf . write ( "<STR_LIT>" % "<STR_LIT:\t>" . join ( ( "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:class>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:start>" , "<STR_LIT:end>" , "<STR_LIT>" ) ) ) <EOL> roi_id = <NUM_LIT:0> <EOL> for gwas , snps in intervals . iteritems ( ) : <EOL> def iter_overlaps ( snps , halfwidth ) : <EOL> snps . sort ( ) <EOL> last_contig , last_pos , last_snp = snps [ <NUM_LIT:0> ] <EOL> width = halfwidth * <NUM_LIT:2> <EOL> r = [ ( last_pos , last_snp ) ] <EOL> for contig , pos , snp in snps [ <NUM_LIT:1> : ] : <EOL> if last_contig != contig or pos - last_pos > width : <EOL> yield last_contig , r <EOL> r = [ ] <EOL> last_contig = contig <EOL> last_pos = pos <EOL> r . append ( ( pos , snp ) ) <EOL> yield last_contig , r <EOL> for contig , overlaps in iter_overlaps ( snps , halfwidth ) : <EOL> pos = [ x [ <NUM_LIT:0> ] for x in overlaps ] <EOL> pos . sort ( ) <EOL> start = pos [ <NUM_LIT:0> ] - halfwidth <EOL> end = pos [ - <NUM_LIT:1> ] + halfwidth <EOL> genes = [ ] <EOL> xsnps = [ x [ <NUM_LIT:1> ] for x in overlaps ] <EOL> for snp in xsnps : <EOL> for gwas , ggenes in ids [ snp ] : <EOL> if ggenes : <EOL> genes . append ( ggenes ) <EOL> genes = sorted ( set ( genes ) ) <EOL> roi_id += <NUM_LIT:1> <EOL> tmpf . write ( "<STR_LIT>" % "<STR_LIT:\t>" . join ( map ( str , <EOL> ( roi_id , <EOL> "<STR_LIT:U+002C>" . join ( xsnps ) , <EOL> gwas , <EOL> contig , <EOL> "<STR_LIT:U+002C>" . join ( map ( str , pos ) ) , <EOL> start , <EOL> end , <EOL> "<STR_LIT:U+002C>" . join ( genes ) , <EOL> ) ) ) ) <EOL> tmpf . close ( ) <EOL> table = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> tmpfilename = tmpf . name <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** locals ( ) ) <EOL> os . unlink ( tmpfilename ) <EOL> outfile = open ( outfile , "<STR_LIT:a>" ) <EOL> outfile . write ( "<STR_LIT>" % ( len ( ids ) , len ( found ) , str ( counter ) ) ) <EOL> outfile . write ( "<STR_LIT>" % "<STR_LIT:U+002C>" . join ( set ( ids . keys ( ) ) . difference ( found ) ) ) <EOL> outfile . close ( ) <EOL> E . info ( "<STR_LIT>" % ( len ( ids ) , len ( found ) , str ( counter ) ) ) <EOL> if len ( ids ) != len ( found ) : <EOL> E . warn ( "<STR_LIT>" % "<STR_LIT:U+002C>" . join ( set ( ids . keys ( ) ) . difference ( found ) ) ) <EOL> @ transform ( ( importRegionsOfInterest , <EOL> importGWAS , <EOL> importMergedGWAS ) , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def importRegionsOfInterestGenes ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> intablename = infile [ : - len ( "<STR_LIT>" ) ] <EOL> tablename = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> E . info ( "<STR_LIT>" % ( intablename , tablename ) ) <EOL> dbhandle = sqlite3 . connect ( PARAMS [ "<STR_LIT>" ] ) <EOL> statement = "<STR_LIT>" <EOL> cc = dbhandle . cursor ( ) <EOL> map_gene2id = dict ( cc . execute ( statement ) . fetchall ( ) ) <EOL> statement = "<STR_LIT>" % locals ( ) <EOL> tmpf = P . getTempFile ( ) <EOL> tmpf . write ( "<STR_LIT>" ) <EOL> cc = dbhandle . cursor ( ) <EOL> counter = E . Counter ( ) <EOL> notfound = set ( ) <EOL> for roi_id , g in cc . execute ( statement ) : <EOL> counter . input += <NUM_LIT:1> <EOL> if g == None : <EOL> counter . nogenes += <NUM_LIT:1> <EOL> continue <EOL> genes = [ x . lower ( ) for x in g . split ( "<STR_LIT:U+002C>" ) ] <EOL> for gene_name in genes : <EOL> if gene_name in map_gene2id : <EOL> counter . found += <NUM_LIT:1> <EOL> tmpf . write ( "<STR_LIT>" % "<STR_LIT:\t>" . join ( map ( str , <EOL> ( roi_id , map_gene2id [ gene_name ] , gene_name ) ) ) ) <EOL> else : <EOL> counter . missed += <NUM_LIT:1> <EOL> notfound . add ( gene_name ) <EOL> E . info ( "<STR_LIT>" % ( str ( counter ) ) ) <EOL> outf = open ( outfile , "<STR_LIT:a>" ) <EOL> outf . write ( "<STR_LIT>" % ( str ( counter ) ) ) <EOL> outf . write ( "<STR_LIT>" % "<STR_LIT:U+002C>" . join ( notfound ) ) <EOL> outf . close ( ) <EOL> tmpf . close ( ) <EOL> tmpfilename = tmpf . name <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> os . unlink ( tmpfilename ) <EOL> @ merge ( PARAMS [ "<STR_LIT>" ] , "<STR_LIT>" ) <EOL> def importSNPsOfInterest ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> table = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ transform ( ( importRegionsOfInterest , importGWAS , importMergedGWAS ) , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def exportRegionsOfInterest ( infile , outfile ) : <EOL> dbhandle = sqlite3 . connect ( PARAMS [ "<STR_LIT>" ] ) <EOL> cc = dbhandle . cursor ( ) <EOL> outs = open ( outfile , "<STR_LIT:w>" ) <EOL> tablename = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> tracks = [ x [ <NUM_LIT:0> ] for x in cc . execute ( "<STR_LIT>" % locals ( ) ) . fetchall ( ) ] <EOL> for track in tracks : <EOL> outs . write ( "<STR_LIT>" % track ) <EOL> statement = "<STR_LIT>" % locals ( ) <EOL> cc . execute ( statement ) <EOL> for contig , start , end , roi_id in cc : <EOL> outs . write ( "<STR_LIT>" % ( contig , start , end , roi_id ) ) <EOL> outs . close ( ) <EOL> @ files ( [ ( "<STR_LIT>" % x , "<STR_LIT>" % x ) for x in TRACKS_REFERENCE ] ) <EOL> def importReferenceIntervals ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> track = infile [ : - len ( "<STR_LIT>" ) ] <EOL> tmpfile = tempfile . NamedTemporaryFile ( delete = False ) <EOL> tmpfilename = tmpfile . name <EOL> headers = ( "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT:start>" , "<STR_LIT:end>" ) <EOL> avgval , contig , disttostart , end , genelist , length , peakcenter , peakval , position , start , interval_id , ncpgs , ngenes , npeaks , nprobes , npromoters = <NUM_LIT:0> , "<STR_LIT>" , <NUM_LIT:0> , <NUM_LIT:0> , "<STR_LIT>" , <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> , <EOL> tmpfile . write ( "<STR_LIT:\t>" . join ( headers ) + "<STR_LIT:\n>" ) <EOL> for line in open ( infile , "<STR_LIT:r>" ) : <EOL> contig , start , end , interval_id = line [ : - <NUM_LIT:1> ] . split ( "<STR_LIT:\t>" ) [ : <NUM_LIT:4> ] <EOL> start , end = int ( start ) , int ( end ) <EOL> length = end - start <EOL> npeaks , peakcenter = <NUM_LIT:1> , start + int ( length / <NUM_LIT:2> ) <EOL> avgval , peakval , nprobes = <NUM_LIT:1> , <NUM_LIT:1> , <NUM_LIT:1> <EOL> tmpfile . write ( "<STR_LIT:\t>" . join ( map ( str , ( avgval , disttostart , genelist , length , peakcenter , peakval , position , interval_id , ncpgs , ngenes , npeaks , nprobes , npromoters , contig , start , end ) ) ) + "<STR_LIT:\n>" ) <EOL> tmpfile . close ( ) <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> os . unlink ( tmpfile . name ) <EOL> if PARAMS [ "<STR_LIT>" ] == "<STR_LIT>" : <EOL> @ files ( PARAMS [ "<STR_LIT>" ] , [ "<STR_LIT>" % x for x in TRACKS_RAW ] ) <EOL> def createIntervals ( infile , outfiles ) : <EOL> '''<STR_LIT>''' <EOL> tmpdir = tempfile . mkdtemp ( ) <EOL> reader = csv . reader ( open ( infile , "<STR_LIT>" ) ) <EOL> output_files_info = IOTools . FilePool ( output_pattern = "<STR_LIT>" ) <EOL> output_files_bed = IOTools . FilePool ( output_pattern = os . path . join ( tmpdir , "<STR_LIT>" ) ) <EOL> for row in reader : <EOL> row = [ re . sub ( "<STR_LIT>" , "<STR_LIT>" , x ) for x in row ] <EOL> row = [ re . sub ( "<STR_LIT>" , "<STR_LIT:n>" , x ) for x in row ] <EOL> if row [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> row = [ re . sub ( "<STR_LIT>" , "<STR_LIT:n>" , x ) for x in row ] <EOL> row = [ re . sub ( "<STR_LIT>" , "<STR_LIT>" , x ) for x in row ] <EOL> row [ <NUM_LIT:1> ] = "<STR_LIT>" <EOL> del row [ <NUM_LIT:2> : <NUM_LIT:5> ] <EOL> output_files_info . setHeader ( "<STR_LIT:\t>" . join ( row [ <NUM_LIT:1> : ] ) + "<STR_LIT:\n>" ) <EOL> else : <EOL> row [ <NUM_LIT:3> ] = str ( int ( row [ <NUM_LIT:3> ] ) - <NUM_LIT:1> ) <EOL> row [ <NUM_LIT:7> ] = str ( int ( row [ <NUM_LIT:7> ] ) - <NUM_LIT:1> ) <EOL> row [ <NUM_LIT:2> ] = "<STR_LIT>" % row [ <NUM_LIT:2> ] <EOL> run_id , cell , condition , replicate = row [ <NUM_LIT:0> ] . split ( "<STR_LIT:_>" ) <EOL> id = "<STR_LIT>" % ( cell , condition , replicate ) <EOL> output_files_bed . write ( id , "<STR_LIT:\t>" . join ( row [ <NUM_LIT:2> : <NUM_LIT:5> ] + row [ <NUM_LIT:1> : <NUM_LIT:2> ] ) + "<STR_LIT:\n>" ) <EOL> del row [ <NUM_LIT:2> : <NUM_LIT:5> ] <EOL> output_files_info . write ( id , "<STR_LIT:\t>" . join ( row [ <NUM_LIT:1> : ] ) + "<STR_LIT:\n>" ) <EOL> output_files_bed . close ( ) <EOL> output_files_info . close ( ) <EOL> if PARAMS [ "<STR_LIT>" ] : <EOL> chain = PARAMS [ "<STR_LIT>" ] <EOL> statement = '''<STR_LIT>''' <EOL> else : <EOL> statement = '''<STR_LIT>''' <EOL> for filename in output_files_bed . keys ( ) : <EOL> basename = os . path . basename ( filename ) <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> shutil . rmtree ( tmpdir ) <EOL> @ transform ( "<STR_LIT>" , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def importIntervals ( infile , outfile ) : <EOL> track = infile [ : - len ( "<STR_LIT>" ) ] <EOL> tmpfile = tempfile . NamedTemporaryFile ( delete = False ) <EOL> tmpfilename = tmpfile . name <EOL> map_id2coords = { } <EOL> for line in open ( track + "<STR_LIT>" , "<STR_LIT:r>" ) : <EOL> contig , start , end , id = line [ : - <NUM_LIT:1> ] . split ( "<STR_LIT:\t>" ) [ : <NUM_LIT:4> ] <EOL> map_id2coords [ id ] = ( contig , start , end ) <EOL> for line in open ( infile ) : <EOL> if line . startswith ( "<STR_LIT>" ) : <EOL> tmpfile . write ( "<STR_LIT>" % ( ( line [ : - <NUM_LIT:1> ] , "<STR_LIT>" , "<STR_LIT:start>" , "<STR_LIT:end>" ) ) ) <EOL> else : <EOL> id = line [ : - <NUM_LIT:1> ] . split ( "<STR_LIT:\t>" ) [ <NUM_LIT:0> ] <EOL> try : <EOL> tmpfile . write ( "<STR_LIT>" % ( ( line [ : - <NUM_LIT:1> ] , ) + map_id2coords [ id ] ) ) <EOL> except KeyError : <EOL> P . warn ( "<STR_LIT>" % ( infile , id ) ) <EOL> tmpfile . close ( ) <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> os . unlink ( tmpfile . name ) <EOL> @ transform ( "<STR_LIT>" , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def exportIntervalsAsBed ( infile , outfile ) : <EOL> _exportIntervalsAsBed ( infile , outfile ) <EOL> @ files_re ( PARAMS [ "<STR_LIT>" ] , <EOL> "<STR_LIT>" , <EOL> r"<STR_LIT>" ) <EOL> def buildBAM ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> tmpfilename = P . getTempFilename ( ) <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> prefix = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> pysam . sort ( tmpfilename , prefix ) <EOL> pysam . index ( outfile ) <EOL> os . unlink ( tmpfilename ) <EOL> @ files_re ( buildBAM , <EOL> "<STR_LIT>" , <EOL> r"<STR_LIT>" ) <EOL> def buildBAMStats ( infile , outfile ) : <EOL> PipelineChipseq . buildBAMStats ( infile , outfile ) <EOL> @ transform ( buildBAMStats , <EOL> regex ( r"<STR_LIT>" ) , <EOL> inputs ( ( r"<STR_LIT>" , r"<STR_LIT>" ) ) , <EOL> r"<STR_LIT>" ) <EOL> def normalizeBAMPerReplicate ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> track = infile [ : - len ( "<STR_LIT>" ) ] <EOL> PipelineChipseq . buildNormalizedBAM ( ( ( infile , track + "<STR_LIT>" ) , ) , outfile ) <EOL> @ follows ( buildBAMStats ) <EOL> @ files ( [ ( [ ( "<STR_LIT>" % ( x , y ) , "<STR_LIT>" % ( x , y ) ) for y in REPLICATES ] , <EOL> "<STR_LIT>" % x ) for x in TRACKS_EXPERIMENTS ] ) <EOL> def normalizeBAMPerExperiment ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . buildNormalizedBAM ( infiles , outfile ) <EOL> @ follows ( exportIntervalsAsBed ) <EOL> @ files ( [ ( [ "<STR_LIT>" % ( y , x ) for y in CELLLINES ] , <EOL> "<STR_LIT>" % x , <EOL> x ) for x in CONDITIONS ] ) <EOL> def combineConditions ( infile , outfile , track ) : <EOL> '''<STR_LIT>''' <EOL> conditions = [ "<STR_LIT>" % ( x , track ) for x in CELLLINES ] <EOL> PipelineChipseq . intersectBedFiles ( conditions , outfile ) <EOL> @ follows ( combineConditions ) <EOL> @ files ( [ ( ( "<STR_LIT>" % x , "<STR_LIT>" ) , <EOL> "<STR_LIT>" % x , <EOL> x ) for x in TRACKS_EXPERIMENTS + TRACKS_CONDITIONS if not re . search ( "<STR_LIT>" , x ) ] ) <EOL> def combineUnstim ( infiles , outfile , track ) : <EOL> '''<STR_LIT>''' <EOL> infile , subtract = infiles <EOL> PipelineChipseq . subtractBedFiles ( infile , subtract , outfile ) <EOL> @ follows ( exportIntervalsAsBed ) <EOL> @ files ( [ ( [ "<STR_LIT>" % ( x , y ) for y in REPLICATES ] , <EOL> "<STR_LIT>" % x , <EOL> x ) for x in TRACKS_EXPERIMENTS ] ) <EOL> def combineReplicates ( infile , outfile , track ) : <EOL> '''<STR_LIT>''' <EOL> replicates = [ "<STR_LIT>" % ( track , x ) for x in REPLICATES if os . path . exists ( "<STR_LIT>" % ( track , x ) ) ] <EOL> PipelineChipseq . intersectBedFiles ( replicates , outfile ) <EOL> @ transform ( ( combineReplicates , <EOL> combineConditions , <EOL> combineUnstim , <EOL> normalizeBAMPerExperiment , <EOL> normalizeBAMPerReplicate ) , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def importCombinedIntervals ( infiles , outfile ) : <EOL> PipelineChipseq . importCombinedIntervals ( infiles , outfile ) <EOL> @ follows ( importCombinedIntervals ) <EOL> def buildIntervals ( ) : <EOL> pass <EOL> elif PARAMS [ "<STR_LIT>" ] == "<STR_LIT>" : <EOL> @ follows ( indexGenome ) <EOL> @ transform ( PARAMS [ "<STR_LIT>" ] , <EOL> regex ( "<STR_LIT>" ) , <EOL> r"<STR_LIT>" ) <EOL> def buildBAM ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> to_cluster = True <EOL> job_options = "<STR_LIT>" <EOL> job_options += "<STR_LIT>" <EOL> tmpfilename = P . getTempFilename ( ) <EOL> prefix = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> if os . path . exists ( tmpfilename ) : <EOL> os . unlink ( tmpfilename ) <EOL> @ transform ( buildBAM , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def buildBAMStats ( infile , outfile ) : <EOL> PipelineChipseq . buildBAMStats ( infile , outfile ) <EOL> @ transform ( buildBAMStats , <EOL> regex ( r"<STR_LIT>" ) , <EOL> inputs ( ( r"<STR_LIT>" , r"<STR_LIT>" ) ) , <EOL> r"<STR_LIT>" ) <EOL> def normalizeBAMPerReplicate ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . buildNormalizedBAM ( ( infiles , ) , outfile ) <EOL> @ transform ( buildBAMStats , <EOL> regex ( r"<STR_LIT>" ) , <EOL> inputs ( ( r"<STR_LIT>" , r"<STR_LIT>" ) ) , <EOL> r"<STR_LIT>" ) <EOL> def normalizeBAMControls ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . buildNormalizedBAM ( ( infiles , ) , outfile ) <EOL> @ follows ( buildBAMStats ) <EOL> @ files ( [ ( [ ( "<STR_LIT>" % ( x , y ) , "<STR_LIT>" % ( x , y ) ) for y in REPLICATES ] , <EOL> "<STR_LIT>" % x ) for x in TRACKS_EXPERIMENTS ] ) <EOL> def normalizeBAMPerExperiment ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . buildNormalizedBAM ( infiles , outfile ) <EOL> @ follows ( normalizeBAMControls ) <EOL> @ transform ( ( normalizeBAMPerReplicate , normalizeBAMPerExperiment ) , <EOL> regex ( "<STR_LIT>" ) , <EOL> r"<STR_LIT>" ) <EOL> def runMACS ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . runMACS ( infile , outfile ) <EOL> @ transform ( runMACS , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def importMACS ( infile , outfile ) : <EOL> PipelineChipseq . importMACS ( infile , outfile ) <EOL> @ merge ( runMACS , "<STR_LIT>" ) <EOL> def summarizeMACS ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . summarizeMACS ( infiles , outfile ) <EOL> @ transform ( summarizeMACS , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def importMACSSummary ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . importMACSSummary ( infile , outfile ) <EOL> @ transform ( importMACS , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def exportIntervalsAsBed ( infile , outfile ) : <EOL> PipelineChipseq . exportIntervalsAsBed ( infile , outfile ) <EOL> @ follows ( exportIntervalsAsBed ) <EOL> @ files ( [ ( [ "<STR_LIT>" % ( y , x ) for y in CELLLINES ] , <EOL> "<STR_LIT>" % x , <EOL> x ) for x in CONDITIONS ] ) <EOL> def combineConditions ( infile , outfile , track ) : <EOL> '''<STR_LIT>''' <EOL> conditions = [ "<STR_LIT>" % ( x , track ) for x in CELLLINES ] <EOL> PipelineChipseq . intersectBedFiles ( conditions , outfile ) <EOL> @ follows ( combineConditions ) <EOL> @ files ( [ ( ( "<STR_LIT>" % x , "<STR_LIT>" ) , <EOL> "<STR_LIT>" % x , <EOL> x ) for x in TRACKS_EXPERIMENTS + TRACKS_CONDITIONS if not re . search ( "<STR_LIT>" , x ) ] ) <EOL> def combineUnstim ( infiles , outfile , track ) : <EOL> '''<STR_LIT>''' <EOL> infile , subtract = infiles <EOL> PipelineChipseq . subtractBedFiles ( infile , subtract , outfile ) <EOL> @ transform ( ( combineConditions , <EOL> combineUnstim , <EOL> normalizeBAMPerExperiment , <EOL> normalizeBAMPerReplicate ) , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def importCombinedIntervals ( infiles , outfile ) : <EOL> PipelineChipseq . importCombinedIntervals ( infiles , outfile ) <EOL> @ follows ( exportIntervalsAsBed ) <EOL> def combineReplicates ( ) : pass <EOL> @ follows ( importCombinedIntervals , summarizeMACS ) <EOL> def buildIntervals ( ) : <EOL> pass <EOL> elif PARAMS [ "<STR_LIT>" ] == "<STR_LIT>" : <EOL> @ follows ( indexGenome ) <EOL> @ transform ( PARAMS [ "<STR_LIT>" ] , <EOL> regex ( "<STR_LIT>" ) , <EOL> r"<STR_LIT>" ) <EOL> def buildBAM ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> tmpfilename = P . getTempFilename ( ) <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> prefix = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> pysam . sort ( tmpfilename , prefix ) <EOL> pysam . index ( outfile ) <EOL> os . unlink ( tmpfilename ) <EOL> @ transform ( buildBAM , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def buildBAMStats ( infile , outfile ) : <EOL> PipelineChipseq . buildBAMStats ( infile , outfile ) <EOL> @ transform ( buildBAMStats , <EOL> regex ( r"<STR_LIT>" ) , <EOL> inputs ( ( r"<STR_LIT>" , r"<STR_LIT>" ) ) , <EOL> r"<STR_LIT>" ) <EOL> def normalizeBAMPerReplicate ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> track = infile [ : - len ( "<STR_LIT>" ) ] <EOL> PipelineChipseq . buildNormalizedBAM ( ( ( infile , track + "<STR_LIT>" ) , ) , outfile ) <EOL> @ transform ( buildBAMStats , <EOL> regex ( r"<STR_LIT>" ) , <EOL> inputs ( ( r"<STR_LIT>" , r"<STR_LIT>" ) ) , <EOL> r"<STR_LIT>" ) <EOL> def normalizeBAMControls ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> track = infile [ : - len ( "<STR_LIT>" ) ] <EOL> PipelineChipseq . buildNormalizedBAM ( ( ( infile , track + "<STR_LIT>" ) , ) , outfile ) <EOL> @ follows ( buildBAMStats ) <EOL> @ files ( [ ( [ ( "<STR_LIT>" % ( x , y ) , "<STR_LIT>" % ( x , y ) ) for y in REPLICATES ] , <EOL> "<STR_LIT>" % x ) for x in TRACKS_EXPERIMENTS ] ) <EOL> def normalizeBAMPerExperiment ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . buildNormalizedBAM ( infiles , outfile ) <EOL> @ follows ( normalizeBAMControls ) <EOL> @ transform ( ( normalizeBAMPerReplicate , normalizeBAMPerExperiment ) , <EOL> regex ( "<STR_LIT>" ) , <EOL> r"<STR_LIT>" ) <EOL> def runMACS ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . runMACS ( infile , outfile ) <EOL> @ transform ( runMACS , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def importMACS ( infile , outfile ) : <EOL> PipelineChipseq . importMACS ( infile , outfile ) <EOL> @ merge ( runMACS , "<STR_LIT>" ) <EOL> def summarizeMACS ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . summarizeMACS ( infiles , outfile ) <EOL> @ transform ( summarizeMACS , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def importMACSSummary ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . importMACSSummary ( infile , outfile ) <EOL> @ transform ( importMACS , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def exportIntervalsAsBed ( infile , outfile ) : <EOL> PipelineChipseq . exportIntervalsAsBed ( infile , outfile ) <EOL> @ follows ( exportIntervalsAsBed ) <EOL> @ files ( [ ( [ "<STR_LIT>" % ( y , x ) for y in CELLLINES ] , <EOL> "<STR_LIT>" % x , <EOL> x ) for x in CONDITIONS ] ) <EOL> def combineConditions ( infile , outfile , track ) : <EOL> '''<STR_LIT>''' <EOL> conditions = [ "<STR_LIT>" % ( x , track ) for x in CELLLINES ] <EOL> PipelineChipseq . intersectBedFiles ( conditions , outfile ) <EOL> @ follows ( combineConditions ) <EOL> @ files ( [ ( ( "<STR_LIT>" % x , "<STR_LIT>" ) , <EOL> "<STR_LIT>" % x , <EOL> x ) for x in TRACKS_EXPERIMENTS + TRACKS_CONDITIONS if not re . search ( "<STR_LIT>" , x ) ] ) <EOL> def combineUnstim ( infiles , outfile , track ) : <EOL> '''<STR_LIT>''' <EOL> infile , subtract = infiles <EOL> PipelineChipseq . subtractBedFiles ( infile , subtract , outfile ) <EOL> @ transform ( ( combineConditions , <EOL> combineUnstim , <EOL> normalizeBAMPerExperiment , <EOL> normalizeBAMPerReplicate ) , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def importCombinedIntervals ( infiles , outfile ) : <EOL> PipelineChipseq . importCombinedIntervals ( infiles , outfile ) <EOL> @ follows ( importCombinedIntervals , importMACSSummary ) <EOL> def buildIntervals ( ) : <EOL> pass <EOL> elif PARAMS [ "<STR_LIT>" ] == "<STR_LIT>" : <EOL> @ follows ( indexGenome ) <EOL> @ transform ( PARAMS [ "<STR_LIT>" ] , <EOL> regex ( "<STR_LIT>" ) , <EOL> r"<STR_LIT>" ) <EOL> def buildBAM ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> to_cluster = True <EOL> job_options = "<STR_LIT>" <EOL> job_options += "<STR_LIT>" <EOL> tmpfilename = P . getTempFilename ( ) <EOL> prefix = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> if os . path . exists ( tmpfilename ) : <EOL> os . unlink ( tmpfilename ) <EOL> @ transform ( buildBAM , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def buildBAMStats ( infile , outfile ) : <EOL> PipelineChipseq . buildBAMStats ( infile , outfile ) <EOL> @ transform ( buildBAMStats , <EOL> regex ( r"<STR_LIT>" ) , <EOL> inputs ( ( r"<STR_LIT>" , r"<STR_LIT>" ) ) , <EOL> r"<STR_LIT>" ) <EOL> def normalizeBAMPerReplicate ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> track = infile [ : - len ( "<STR_LIT>" ) ] <EOL> PipelineChipseq . buildNormalizedBAM ( ( ( infile , track + "<STR_LIT>" ) , ) , outfile ) <EOL> @ follows ( buildBAMStats ) <EOL> @ files ( [ ( [ ( "<STR_LIT>" % ( x , y ) , "<STR_LIT>" % ( x , y ) ) for y in REPLICATES ] , <EOL> "<STR_LIT>" % x ) for x in TRACKS_EXPERIMENTS ] ) <EOL> def normalizeBAMPerExperiment ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . buildNormalizedBAM ( infiles , outfile ) <EOL> @ transform ( buildBAMStats , <EOL> regex ( r"<STR_LIT>" ) , <EOL> inputs ( ( r"<STR_LIT>" , r"<STR_LIT>" ) ) , <EOL> r"<STR_LIT>" ) <EOL> def normalizeBAMControls ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> track = infile [ : - len ( "<STR_LIT>" ) ] <EOL> PipelineChipseq . buildNormalizedBAM ( ( ( infile , track + "<STR_LIT>" ) , ) , outfile ) <EOL> @ follows ( normalizeBAMControls ) <EOL> @ transform ( normalizeBAMPerReplicate , <EOL> regex ( "<STR_LIT>" ) , <EOL> r"<STR_LIT>" ) <EOL> def runMACS ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . runMACS ( infile , outfile ) <EOL> @ transform ( runMACS , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def importMACS ( infile , outfile ) : <EOL> PipelineChipseq . importMACS ( infile , outfile ) <EOL> @ merge ( runMACS , "<STR_LIT>" ) <EOL> def summarizeMACS ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . summarizeMACS ( infiles , outfile ) <EOL> @ transform ( summarizeMACS , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def importMACSSummary ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . importMACSSummary ( infile , outfile ) <EOL> @ transform ( importMACS , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def exportIntervalsAsBed ( infile , outfile ) : <EOL> PipelineChipseq . exportIntervalsAsBed ( infile , outfile ) <EOL> @ follows ( exportIntervalsAsBed ) <EOL> @ files ( [ ( [ "<STR_LIT>" % ( x , y ) for y in REPLICATES ] , <EOL> "<STR_LIT>" % x , <EOL> x ) for x in TRACKS_EXPERIMENTS ] ) <EOL> def combineReplicates ( infile , outfile , track ) : <EOL> '''<STR_LIT>''' <EOL> replicates = [ "<STR_LIT>" % ( track , x ) for x in REPLICATES if os . path . exists ( "<STR_LIT>" % ( track , x ) ) ] <EOL> PipelineChipseq . intersectBedFiles ( replicates , outfile ) <EOL> @ follows ( exportIntervalsAsBed , combineReplicates ) <EOL> @ files ( [ ( [ "<STR_LIT>" % ( y , x ) for y in CELLLINES ] , <EOL> "<STR_LIT>" % x , <EOL> x ) for x in CONDITIONS ] ) <EOL> def combineConditions ( infile , outfile , track ) : <EOL> '''<STR_LIT>''' <EOL> conditions = [ "<STR_LIT>" % ( x , track ) for x in CELLLINES ] <EOL> PipelineChipseq . intersectBedFiles ( conditions , outfile ) <EOL> @ follows ( combineConditions ) <EOL> @ files ( [ ( ( "<STR_LIT>" % x , "<STR_LIT>" ) , <EOL> "<STR_LIT>" % x , <EOL> x ) for x in TRACKS_EXPERIMENTS + TRACKS_CONDITIONS if not re . search ( "<STR_LIT>" , x ) ] ) <EOL> def combineUnstim ( infiles , outfile , track ) : <EOL> '''<STR_LIT>''' <EOL> infile , subtract = infiles <EOL> PipelineChipseq . subtractBedFiles ( infile , subtract , outfile ) <EOL> @ transform ( ( combineReplicates , <EOL> combineConditions , <EOL> combineUnstim , <EOL> normalizeBAMPerReplicate ) , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def importCombinedIntervals ( infiles , outfile ) : <EOL> PipelineChipseq . importCombinedIntervals ( infiles , outfile ) <EOL> @ follows ( importCombinedIntervals , importMACSSummary ) <EOL> def buildIntervals ( ) : <EOL> pass <EOL> elif PARAMS [ "<STR_LIT>" ] == "<STR_LIT>" : <EOL> @ follows ( indexGenome ) <EOL> @ transform ( PARAMS [ "<STR_LIT>" ] , <EOL> regex ( "<STR_LIT>" ) , <EOL> r"<STR_LIT>" ) <EOL> def buildBAM ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> to_cluster = True <EOL> job_options = "<STR_LIT>" <EOL> job_options += "<STR_LIT>" <EOL> tmpfilename = P . getTempFilename ( ) <EOL> prefix = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> if os . path . exists ( tmpfilename ) : <EOL> os . unlink ( tmpfilename ) <EOL> @ transform ( buildBAM , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def buildBAMStats ( infile , outfile ) : <EOL> PipelineChipseq . buildBAMStats ( infile , outfile ) <EOL> @ transform ( buildBAMStats , <EOL> regex ( r"<STR_LIT>" ) , <EOL> inputs ( ( r"<STR_LIT>" , r"<STR_LIT>" ) ) , <EOL> r"<STR_LIT>" ) <EOL> def normalizeBAMPerReplicate ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> track = infile [ : - len ( "<STR_LIT>" ) ] <EOL> PipelineChipseq . buildNormalizedBAM ( ( ( infile , track + "<STR_LIT>" ) , ) , outfile ) <EOL> @ follows ( buildBAMStats ) <EOL> @ files ( [ ( [ ( "<STR_LIT>" % ( x , y ) , "<STR_LIT>" % ( x , y ) ) for y in REPLICATES ] , <EOL> "<STR_LIT>" % x ) for x in TRACKS_EXPERIMENTS ] ) <EOL> def normalizeBAMPerExperiment ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . buildNormalizedBAM ( infiles , outfile ) <EOL> @ transform ( buildBAMStats , <EOL> regex ( r"<STR_LIT>" ) , <EOL> inputs ( ( r"<STR_LIT>" , r"<STR_LIT>" ) ) , <EOL> r"<STR_LIT>" ) <EOL> def normalizeBAMControls ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> track = infile [ : - len ( "<STR_LIT>" ) ] <EOL> PipelineChipseq . buildNormalizedBAM ( ( ( infile , track + "<STR_LIT>" ) , ) , outfile ) <EOL> @ follows ( normalizeBAMControls ) <EOL> @ transform ( normalizeBAMPerReplicate , <EOL> regex ( "<STR_LIT>" ) , <EOL> r"<STR_LIT>" ) <EOL> def runMACS ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . runMACS ( infile , outfile ) <EOL> @ transform ( runMACS , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def importMACS ( infile , outfile ) : <EOL> PipelineChipseq . importMACS ( infile , outfile ) <EOL> @ merge ( runMACS , "<STR_LIT>" ) <EOL> def summarizeMACS ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . summarizeMACS ( infiles , outfile ) <EOL> @ transform ( summarizeMACS , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def importMACSSummary ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . importMACSSummary ( infile , outfile ) <EOL> @ transform ( importMACS , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def exportIntervalsAsBed ( infile , outfile ) : <EOL> PipelineChipseq . exportPeaksAsBed ( infile , outfile ) <EOL> @ follows ( exportIntervalsAsBed ) <EOL> @ files ( [ ( [ "<STR_LIT>" % ( x , y ) for y in REPLICATES ] , <EOL> "<STR_LIT>" % x , <EOL> x ) for x in TRACKS_EXPERIMENTS ] ) <EOL> def combineReplicates ( infile , outfile , track ) : <EOL> '''<STR_LIT>''' <EOL> replicates = [ "<STR_LIT>" % ( track , x ) for x in REPLICATES if os . path . exists ( "<STR_LIT>" % ( track , x ) ) ] <EOL> PipelineChipseq . intersectBedFiles ( replicates , outfile ) <EOL> @ follows ( exportIntervalsAsBed , combineReplicates ) <EOL> @ files ( [ ( [ "<STR_LIT>" % ( y , x ) for y in CELLLINES ] , <EOL> "<STR_LIT>" % x , <EOL> x ) for x in CONDITIONS ] ) <EOL> def combineConditions ( infile , outfile , track ) : <EOL> '''<STR_LIT>''' <EOL> conditions = [ "<STR_LIT>" % ( x , track ) for x in CELLLINES ] <EOL> PipelineChipseq . intersectBedFiles ( conditions , outfile ) <EOL> @ follows ( combineConditions ) <EOL> @ files ( [ ( ( "<STR_LIT>" % x , "<STR_LIT>" ) , <EOL> "<STR_LIT>" % x , <EOL> x ) for x in TRACKS_EXPERIMENTS + TRACKS_CONDITIONS if not re . search ( "<STR_LIT>" , x ) ] ) <EOL> def combineUnstim ( infiles , outfile , track ) : <EOL> '''<STR_LIT>''' <EOL> infile , subtract = infiles <EOL> PipelineChipseq . subtractBedFiles ( infile , subtract , outfile ) <EOL> @ transform ( ( combineReplicates , <EOL> combineConditions , <EOL> combineUnstim , <EOL> normalizeBAMPerReplicate ) , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def importCombinedIntervals ( infiles , outfile ) : <EOL> PipelineChipseq . importCombinedIntervals ( infiles , outfile ) <EOL> @ follows ( importCombinedIntervals , importMACSSummary ) <EOL> def buildIntervals ( ) : <EOL> pass <EOL> elif PARAMS [ "<STR_LIT>" ] == "<STR_LIT>" : <EOL> BAM_SUFFIX = "<STR_LIT>" <EOL> @ transform ( "<STR_LIT>" , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def buildBAM ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> tmpfilename1 = P . getTempFilename ( ) <EOL> tmpfilename2 = P . getTempFilename ( ) <EOL> outf = open ( tmpfilename1 , "<STR_LIT:w>" ) <EOL> fasta = IndexedFasta . IndexedFasta ( PARAMS [ "<STR_LIT>" ] ) <EOL> for x , b in enumerate ( Bed . iterator ( gzip . open ( infile ) ) ) : <EOL> l = b . end - b . start <EOL> strand = b . fields [ <NUM_LIT:2> ] <EOL> if strand == "<STR_LIT:+>" : flag = "<STR_LIT:0>" <EOL> elif strand == "<STR_LIT:->" : flag = "<STR_LIT>" <EOL> else : raise ValueError ( "<STR_LIT>" % str ( b ) ) <EOL> outf . write ( "<STR_LIT:\t>" . join ( <EOL> ( "<STR_LIT>" % x , <EOL> flag , <EOL> b . contig , <EOL> str ( b . start + <NUM_LIT:1> ) , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" % l , <EOL> "<STR_LIT:*>" , "<STR_LIT:0>" , "<STR_LIT:0>" , <EOL> "<STR_LIT:A>" * l , <EOL> "<STR_LIT:A>" * l , <EOL> "<STR_LIT>" % l ) ) + "<STR_LIT:\n>" ) <EOL> outf . close ( ) <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> prefix = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> pysam . sort ( tmpfilename2 , prefix ) <EOL> pysam . index ( outfile ) <EOL> os . unlink ( tmpfilename1 ) <EOL> os . unlink ( tmpfilename2 ) <EOL> @ transform ( buildBAM , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def runMACS ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . runMACS ( infile , outfile ) <EOL> @ transform ( runMACS , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def importMACS ( infile , outfile ) : <EOL> PipelineChipseq . importMACS ( infile , outfile , suffix = "<STR_LIT>" ) <EOL> @ merge ( runMACS , "<STR_LIT>" ) <EOL> def summarizeMACS ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . summarizeMACS ( infiles , outfile ) <EOL> @ transform ( summarizeMACS , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def importMACSSummary ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . importMACSSummary ( infile , outfile ) <EOL> @ transform ( importMACS , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def exportIntervalsAsBed ( infile , outfile ) : <EOL> PipelineChipseq . exportIntervalsAsBed ( infile , outfile ) <EOL> @ follows ( exportIntervalsAsBed ) <EOL> def combineReplicates ( ) : pass <EOL> @ follows ( exportIntervalsAsBed , combineReplicates ) <EOL> def combineConditions ( ) : pass <EOL> @ follows ( combineConditions ) <EOL> def combineUnstim ( ) : pass <EOL> @ follows ( exportIntervalsAsBed ) <EOL> def normalizeBAMPerExperiment ( ) : pass <EOL> @ follows ( exportIntervalsAsBed ) <EOL> def normalizeBAMPerReplicate ( ) : pass <EOL> @ transform ( ( combineReplicates , <EOL> combineConditions , <EOL> combineUnstim ) , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def importCombinedIntervals ( infiles , outfile ) : <EOL> PipelineChipseq . importCombinedIntervals ( infiles , outfile ) <EOL> @ follows ( exportIntervalsAsBed , <EOL> importCombinedIntervals , <EOL> importMACSSummary ) <EOL> def buildIntervals ( ) : pass <EOL> @ follows ( exportIntervalsAsBed ) <EOL> @ files_re ( [ "<STR_LIT>" % x for x in TRACKS_RAW ] , <EOL> combine ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def makeMerged ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . mergeBedFiles ( infiles , outfile ) <EOL> @ files ( ( <EOL> ( ( PARAMS [ "<STR_LIT>" ] , <EOL> PARAMS [ "<STR_LIT>" ] , <EOL> PARAMS [ "<STR_LIT>" ] ) , TARGET_TRANSCRIPTS ) , <EOL> ) , ) <EOL> def buildTranscripts ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> map_ensembl2refseq , ensembl , refseq = infiles <EOL> to_cluster = True <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> statement = '''<STR_LIT>''' <EOL> @ files ( PARAMS [ "<STR_LIT>" ] , TARGET_ANNOTATOR_GENETERRITORIES ) <EOL> def buildAnnotatorGeneTerritories ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> to_cluster = True <EOL> radius = PARAMS [ '<STR_LIT>' ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> @ files_re ( buildTranscripts , "<STR_LIT>" , r"<STR_LIT>" ) <EOL> def importTranscripts ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> table = "<STR_LIT>" <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> @ files ( PARAMS [ "<STR_LIT>" ] , TARGET_REPEATS ) <EOL> def buildRepeats ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> instream = gzip . open ( infile ) <EOL> tmpfile = tempfile . NamedTemporaryFile ( delete = False ) <EOL> tmpfilename = tmpfile . name <EOL> gff = GFF . Entry ( ) <EOL> gff . feature = "<STR_LIT>" <EOL> gff . source = "<STR_LIT>" <EOL> for line in instream : <EOL> if line . startswith ( "<STR_LIT>" ) : continue <EOL> ( bin , swScore , milliDiv , milliDel , milliIns , genoName , genoStart , genoEnd , genoLeft , strand , repName , repClass , repFamily , repStart , repEnd , repLeft , id ) = line [ : - <NUM_LIT:1> ] . split ( "<STR_LIT:\t>" ) <EOL> if repClass not in REPEATS : continue <EOL> gff . strand = strand <EOL> gff . contig = genoName <EOL> gff . start = int ( genoStart ) - <NUM_LIT:1> <EOL> gff . end = int ( genoEnd ) <EOL> gff . addAttribute ( "<STR_LIT:name>" , repName ) <EOL> gff . addAttribute ( "<STR_LIT:class>" , repClass ) <EOL> gff . addAttribute ( "<STR_LIT>" , repFamily ) <EOL> tmpfile . write ( str ( gff ) + "<STR_LIT:\n>" ) <EOL> tmpfile . close ( ) <EOL> E . info ( "<STR_LIT>" ) <EOL> statement = """<STR_LIT>""" <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> os . unlink ( tmpfile . name ) <EOL> @ files ( PARAMS [ "<STR_LIT>" ] , TARGET_ANNOTATION ) <EOL> def buildGeneRegions ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PGeneset . buildGeneRegions ( infile , outfile , only_protein_coding = True ) <EOL> @ files ( PARAMS [ "<STR_LIT>" ] , TARGET_GENESET ) <EOL> def buildGeneSet ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PGeneset . buildGeneSet ( infile , outfile , only_protein_coding = True ) <EOL> @ files ( buildGeneSet , "<STR_LIT>" ) <EOL> def importGeneStats ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PGeneset . importGeneStats ( infile , outfile ) <EOL> @ files ( [ ( "<STR_LIT>" % PARAMS [ "<STR_LIT>" ] , "<STR_LIT>" ) , ] ) <EOL> def buildGenome ( infile , outfile ) : <EOL> fasta = IndexedFasta . IndexedFasta ( PARAMS [ "<STR_LIT>" ] ) <EOL> entry = GFF . Entry ( ) <EOL> entry . start = <NUM_LIT:0> <EOL> entry . feature = "<STR_LIT>" <EOL> entry . source = "<STR_LIT>" <EOL> outs = open ( outfile , "<STR_LIT:w>" ) <EOL> for contig , size in fasta . getContigSizes ( with_synonyms = False ) . iteritems ( ) : <EOL> entry . contig = contig <EOL> entry . end = int ( size ) <EOL> outs . write ( "<STR_LIT>" % str ( entry ) ) <EOL> outs . close ( ) <EOL> @ follows ( buildTranscripts ) <EOL> @ files ( ( ( PARAMS [ "<STR_LIT>" ] , TARGET_PROMOTORS ) , <EOL> ( TARGET_TRANSCRIPTS , TARGET_TRANSCRIPTS_PROMOTORS ) , ) ) <EOL> def buildPromotorRegions ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> statement = """<STR_LIT>""" <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> @ follows ( buildTranscripts ) <EOL> @ files ( ( ( PARAMS [ "<STR_LIT>" ] , TARGET_TSS ) , <EOL> ( TARGET_TRANSCRIPTS , TARGET_TRANSCRIPTS_TSS ) , ) ) <EOL> def buildTSSRegions ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> statement = """<STR_LIT>""" <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> @ follows ( buildPromotorRegions , buildGeneRegions , buildTSSRegions ) <EOL> @ files ( [ ( "<STR_LIT>" % x , "<STR_LIT>" % x ) for x in ( "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ) ] ) <EOL> def exportReferenceAsBed ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> bed = Bed . Bed ( ) <EOL> outfile = open ( outfile , "<STR_LIT:w>" ) <EOL> with open ( infile , "<STR_LIT:r>" ) as inf : <EOL> for gff in GTF . iterator ( infile ) : <EOL> bed . contig , bed . start , bed . end = gff . contig , gff . start , gff . end <EOL> bed . fields = [ gff . gene_id ] <EOL> outfile . write ( "<STR_LIT>" % str ( bed ) ) <EOL> outfile . close ( ) <EOL> @ files_re ( ( buildBAM , normalizeBAMPerExperiment , normalizeBAMPerReplicate ) , <EOL> "<STR_LIT>" , <EOL> r"<STR_LIT>" ) <EOL> def buildBigwig ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> to_cluster = True <EOL> job_queue = "<STR_LIT>" <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ files ( ( ( PARAMS [ "<STR_LIT>" ] , "<STR_LIT>" ) , ) ) <EOL> def importAffymetrixAnnotation ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> table = "<STR_LIT>" <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ files ( importAffymetrixAnnotation , "<STR_LIT>" ) <EOL> def buildProbeset2Transcript ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> dbhandle = sqlite3 . connect ( PARAMS [ "<STR_LIT>" ] ) <EOL> E . debug ( "<STR_LIT>" ) <EOL> raise NotImplementError ( "<STR_LIT>" ) <EOL> with gzip . open ( PARAMS [ "<STR_LIT>" ] ) as inf : <EOL> exons = GTF . readAndIndex ( GTF . iterator ( inf ) ) <EOL> E . debug ( "<STR_LIT>" ) <EOL> cc = dbhandle . cursor ( ) <EOL> statement = "<STR_LIT>" <EOL> cc . execute ( statement ) <EOL> map_transcript2gene = { } <EOL> for transcript_id , gene_id , source , contig , start , end , strand in cc . fetchall ( ) : <EOL> map_transcript2gene [ transcript_id ] = ( gene_id , source , contig , start , end , strand ) <EOL> counter = E . Counter ( ) <EOL> cc = dbhandle . cursor ( ) <EOL> statement = "<STR_LIT>" <EOL> cc . execute ( statement ) <EOL> outf_notfound = open ( outfile + "<STR_LIT>" , "<STR_LIT:w>" ) <EOL> outf_notfound . write ( "<STR_LIT>" ) <EOL> outf = open ( outfile , "<STR_LIT:w>" ) <EOL> outf . write ( "<STR_LIT>" ) <EOL> genes_found , transcripts_found = set ( ) , set ( ) <EOL> for probeset , transcript_cluster_id , contig , start , end , mrna_assignment , crosshyb_type in cc . fetchall ( ) : <EOL> start -= <NUM_LIT:1> <EOL> if mrna_assignment == "<STR_LIT>" : <EOL> counter . probesets_unassigned += <NUM_LIT:1> <EOL> continue <EOL> data = [ x . strip ( ) . split ( "<STR_LIT>" ) for x in mrna_assignment . split ( "<STR_LIT>" ) ] <EOL> found , mapped , mismapped , nxhyb = <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> <EOL> try : <EOL> overlaps = [ x [ <NUM_LIT:2> ] for x in exons . get ( contig , start , end ) ] <EOL> except KeyError : <EOL> overlaps = [ ] <EOL> transcripts = set ( [ x . transcript_id for x in overlaps ] ) <EOL> for transcript_id , seqname , score , direct_probes , possible_probes , xhyb in data : <EOL> if xhyb != "<STR_LIT:0>" : <EOL> nxhyb += <NUM_LIT:1> <EOL> continue <EOL> if transcript_id . startswith ( "<STR_LIT>" ) : <EOL> found += <NUM_LIT:1> <EOL> try : <EOL> gene_id , other_source , other_contig , other_start , other_end , other_strand = map_transcript2gene [ transcript_id ] <EOL> except KeyError : <EOL> continue <EOL> mapped += <NUM_LIT:1> <EOL> if transcript_id not in transcripts : <EOL> mismapped += <NUM_LIT:1> <EOL> continue <EOL> if other_contig != contig or min ( other_end , end ) - max ( other_start , start ) < <NUM_LIT:0> : <EOL> mismapped += <NUM_LIT:1> <EOL> continue <EOL> outf . write ( "<STR_LIT>" % ( probeset , transcript_cluster_id , contig , start , end , <EOL> transcript_id , gene_id , other_source , <EOL> other_contig , other_start , other_end , other_strand ) ) <EOL> genes_found . add ( gene_id ) <EOL> transcripts_found . add ( transcript_id ) <EOL> if found == <NUM_LIT:0> : <EOL> counter . probesets_notfound += <NUM_LIT:1> <EOL> code = "<STR_LIT>" <EOL> elif mapped == <NUM_LIT:0> : <EOL> counter . probesets_unmapped += <NUM_LIT:1> <EOL> code = "<STR_LIT>" <EOL> elif mapped > <NUM_LIT:0> and mapped == mismapped : <EOL> counter . probesets_mismapped += <NUM_LIT:1> <EOL> code = "<STR_LIT>" <EOL> elif mapped > <NUM_LIT:0> and mapped == nxhyb : <EOL> counter . probesets_crosshyb += <NUM_LIT:1> <EOL> code = "<STR_LIT>" <EOL> elif mapped > <NUM_LIT:0> and mapped == nxhyb + mismapped : <EOL> counter . probesets_errors += <NUM_LIT:1> <EOL> code = "<STR_LIT>" <EOL> else : <EOL> counter . probesets_found += <NUM_LIT:1> <EOL> code = None <EOL> if code : <EOL> outf_notfound . write ( "<STR_LIT>" % ( probeset , code , contig , start , end , mrna_assignment ) ) <EOL> outf_notfound . close ( ) <EOL> outf . close ( ) <EOL> E . info ( "<STR_LIT>" % str ( counter ) ) <EOL> E . info ( "<STR_LIT>" % ( len ( genes_found ) , <EOL> <NUM_LIT> * len ( genes_found ) / len ( set ( [ x [ <NUM_LIT:0> ] for x in map_transcript2gene . values ( ) ] ) ) , <EOL> len ( transcripts_found ) , <EOL> <NUM_LIT> * len ( transcripts_found ) / len ( map_transcript2gene . keys ( ) ) ) ) <EOL> @ transform ( buildProbeset2Transcript , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def importProbeset2Transcript ( infile , outfile ) : <EOL> table = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ files ( ( ( importProbeset2Transcript , TARGET_PROBESET ) , ) ) <EOL> def exportProbesetLocations ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> dbhandle = sqlite3 . connect ( PARAMS [ "<STR_LIT>" ] ) <EOL> cc = dbhandle . cursor ( ) <EOL> statement = "<STR_LIT>" <EOL> cc . execute ( statement ) <EOL> gff = GTF . Entry ( ) <EOL> gff . source = "<STR_LIT>" <EOL> gff . feature = "<STR_LIT>" <EOL> outs = open ( outfile , "<STR_LIT:w>" ) <EOL> noutput , nskipped = <NUM_LIT:0> , <NUM_LIT:0> <EOL> max_len = int ( PARAMS [ "<STR_LIT>" ] ) <EOL> for contig , ncontigs , start , end , strand , cluster_id , transcript_id , gene_id , ngenes in cc : <EOL> if ncontigs > <NUM_LIT:1> : <EOL> E . warn ( "<STR_LIT>" % cluster_id ) <EOL> nskipped += <NUM_LIT:1> <EOL> continue <EOL> if ngenes > <NUM_LIT:1> : <EOL> E . warn ( "<STR_LIT>" % cluster_id ) <EOL> nskipped += <NUM_LIT:1> <EOL> continue <EOL> if end - start > max_len : <EOL> E . warn ( "<STR_LIT>" % ( cluster_id , contig , start , end , end - start , max_len ) ) <EOL> nskipped += <NUM_LIT:1> <EOL> continue <EOL> gff . contig = contig <EOL> gff . start = start <EOL> gff . end = end <EOL> gff . strand = strand <EOL> gff . gene_id = transcript_id <EOL> gff . transcript_id = cluster_id <EOL> outs . write ( str ( gff ) + "<STR_LIT:\n>" ) <EOL> noutput += <NUM_LIT:1> <EOL> outs . close ( ) <EOL> E . info ( "<STR_LIT>" % ( noutput , nskipped ) ) <EOL> @ transform ( exportProbesetLocations , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def importProbesetLocations ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> tablename = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> @ split ( ( PARAMS [ "<STR_LIT>" ] , PARAMS [ "<STR_LIT>" ] , importProbeset2Transcript ) , <EOL> ( "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ) ) <EOL> def buildExpressionTracks ( infiles , outfiles ) : <EOL> '''<STR_LIT>''' <EOL> for f in outfiles : <EOL> try : <EOL> os . remove ( f ) <EOL> except OSError : <EOL> pass <EOL> outf = open ( "<STR_LIT>" , "<STR_LIT:w>" ) <EOL> outf . write ( "<STR_LIT>" ) <EOL> for index , infile in enumerate ( infiles ) : <EOL> map_exp2columns = collections . defaultdict ( list ) <EOL> if infile == PARAMS [ "<STR_LIT>" ] : <EOL> map_exp2columns = { '<STR_LIT>' : ( <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:5> ) , <EOL> '<STR_LIT>' : ( <NUM_LIT:6> , <NUM_LIT:7> , <NUM_LIT:8> ) , <EOL> '<STR_LIT>' : ( <NUM_LIT:9> , <NUM_LIT:10> , <NUM_LIT:11> ) , <EOL> '<STR_LIT>' : ( <NUM_LIT:12> , <NUM_LIT> , <NUM_LIT> ) , <EOL> '<STR_LIT>' : ( <NUM_LIT:15> , <NUM_LIT:16> , <NUM_LIT> ) , <EOL> '<STR_LIT>' : ( <NUM_LIT> , <NUM_LIT> , <NUM_LIT:20> ) , <EOL> '<STR_LIT>' : ( <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) , <EOL> '<STR_LIT>' : ( <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) , <EOL> '<STR_LIT>' : ( <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:15> , <NUM_LIT:16> , <NUM_LIT> ) , <EOL> '<STR_LIT>' : ( <NUM_LIT:6> , <NUM_LIT:7> , <NUM_LIT:8> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT:20> ) , <EOL> '<STR_LIT>' : ( <NUM_LIT:9> , <NUM_LIT:10> , <NUM_LIT:11> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) , <EOL> '<STR_LIT>' : ( <NUM_LIT:12> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) , <EOL> } <EOL> PExpression . buildExpressionTracks ( infile , outfiles , map_exp2columns , suffix = "<STR_LIT>" ) <EOL> for key in map_exp2columns . keys ( ) : <EOL> cellline , time , stimulus = key . split ( "<STR_LIT:_>" ) <EOL> outf . write ( "<STR_LIT:\t>" . join ( ( key , cellline , time , stimulus ) ) + "<STR_LIT:\n>" ) <EOL> elif infile == PARAMS [ "<STR_LIT>" ] : <EOL> x = <NUM_LIT:1> <EOL> stimulus = "<STR_LIT>" <EOL> for cell in ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) : <EOL> for timepoint in ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) : <EOL> key = "<STR_LIT>" % ( cell , timepoint , stimulus ) <EOL> map_exp2columns [ key ] = ( x , x + <NUM_LIT:1> ) <EOL> if cell . startswith ( "<STR_LIT>" ) : <EOL> key2 = "<STR_LIT>" % ( "<STR_LIT>" , timepoint , stimulus ) <EOL> else : <EOL> key2 = "<STR_LIT>" % ( "<STR_LIT>" , timepoint , stimulus ) <EOL> map_exp2columns [ key2 ] . extend ( [ x , x + <NUM_LIT:1> ] ) <EOL> x += <NUM_LIT:2> <EOL> PExpression . buildExpressionTracks ( infile , outfiles , map_exp2columns , suffix = "<STR_LIT>" % index ) <EOL> for key in map_exp2columns . keys ( ) : <EOL> cellline , time , stimulus = key . split ( "<STR_LIT:_>" ) <EOL> outf . write ( "<STR_LIT:\t>" . join ( ( key , cellline , time , stimulus ) ) + "<STR_LIT:\n>" ) <EOL> outf . close ( ) <EOL> @ transform ( buildExpressionTracks , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def importExpressionTracks ( infile , outfile ) : <EOL> table = outfile [ : - len ( "<STR_LIT>" ) ] + "<STR_LIT>" <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ merge ( buildExpressionTracks , "<STR_LIT>" ) <EOL> def buildExpressionCorrelation ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> outf = open ( outfile , "<STR_LIT:w>" ) <EOL> outf . write ( "<STR_LIT>" % "<STR_LIT:\t>" . join ( Stats . CorrelationTest ( ) . getHeaders ( ) ) ) <EOL> offset = <NUM_LIT:10> <EOL> for filename in infiles : <EOL> if not filename . endswith ( "<STR_LIT>" ) : continue <EOL> vals = [ ] <EOL> headers = [ ] <EOL> track = filename [ : - len ( "<STR_LIT>" ) ] <EOL> inf = open ( filename , "<STR_LIT:r>" ) <EOL> for line in inf : <EOL> if line . startswith ( "<STR_LIT:#>" ) : continue <EOL> data = line [ : - <NUM_LIT:1> ] . split ( "<STR_LIT:\t>" ) <EOL> if line . startswith ( "<STR_LIT>" ) : <EOL> nreplicates = len ( data ) - offset <EOL> for x in range ( nreplicates ) : <EOL> headers . append ( data [ offset + x ] ) <EOL> vals . append ( [ ] ) <EOL> continue <EOL> for x in range ( <NUM_LIT:0> , nreplicates ) : <EOL> vals [ x ] . append ( float ( data [ offset + x ] ) ) <EOL> for x in range ( nreplicates - <NUM_LIT:1> ) : <EOL> for y in range ( x + <NUM_LIT:1> , nreplicates ) : <EOL> outf . write ( "<STR_LIT:\t>" . join ( ( track , <EOL> headers [ x ] , <EOL> headers [ y ] , <EOL> str ( Stats . doCorrelationTest ( vals [ x ] , <EOL> vals [ y ] ) ) ) ) + "<STR_LIT:\n>" ) <EOL> outf . close ( ) <EOL> @ merge ( buildExpressionTracks , "<STR_LIT>" ) <EOL> def buildExpressionFullCorrelation ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> outf = open ( outfile , "<STR_LIT:w>" ) <EOL> outf . write ( "<STR_LIT>" % "<STR_LIT:\t>" . join ( Stats . CorrelationTest ( ) . getHeaders ( ) ) ) <EOL> nfiles = len ( infiles ) <EOL> def _readVals ( filename ) : <EOL> '''<STR_LIT>''' <EOL> vals = [ ] <EOL> headers = [ ] <EOL> offset = <NUM_LIT:10> <EOL> lines = open ( filename , "<STR_LIT:r>" ) . readlines ( ) <EOL> data = lines [ <NUM_LIT:0> ] [ : - <NUM_LIT:1> ] . split ( "<STR_LIT:\t>" ) <EOL> nreplicates = len ( data ) - offset <EOL> for x in range ( nreplicates ) : <EOL> headers . append ( data [ offset + x ] ) <EOL> vals . append ( [ ] ) <EOL> del lines [ <NUM_LIT:0> ] <EOL> lines . sort ( ) <EOL> for line in lines : <EOL> data = line [ : - <NUM_LIT:1> ] . split ( "<STR_LIT:\t>" ) <EOL> for x in range ( <NUM_LIT:0> , nreplicates ) : <EOL> vals [ x ] . append ( float ( data [ offset + x ] ) ) <EOL> return nreplicates , vals , headers <EOL> for f1 in range ( nfiles - <NUM_LIT:1> ) : <EOL> filename1 = infiles [ f1 ] <EOL> if not filename1 . endswith ( "<STR_LIT>" ) : continue <EOL> track1 = filename1 [ : - len ( "<STR_LIT>" ) ] <EOL> nreplicates1 , vals1 , headers1 = _readVals ( filename1 ) <EOL> for x in range ( nreplicates1 - <NUM_LIT:1> ) : <EOL> for y in range ( x + <NUM_LIT:1> , nreplicates1 ) : <EOL> outf . write ( "<STR_LIT:\t>" . join ( ( track1 , <EOL> headers1 [ x ] , <EOL> track1 , <EOL> headers1 [ y ] , <EOL> str ( Stats . doCorrelationTest ( vals1 [ x ] , <EOL> vals1 [ y ] ) ) ) ) + "<STR_LIT:\n>" ) <EOL> for f2 in range ( f1 + <NUM_LIT:1> , nfiles ) : <EOL> filename2 = infiles [ f2 ] <EOL> if not filename2 . endswith ( "<STR_LIT>" ) : continue <EOL> track2 = filename2 [ : - len ( "<STR_LIT>" ) ] <EOL> nreplicates2 , vals2 , headers2 = _readVals ( filename2 ) <EOL> for x in range ( nreplicates1 ) : <EOL> for y in range ( nreplicates2 ) : <EOL> outf . write ( "<STR_LIT:\t>" . join ( ( track1 , <EOL> headers1 [ x ] , <EOL> track2 , <EOL> headers2 [ y ] , <EOL> str ( Stats . doCorrelationTest ( vals1 [ x ] , <EOL> vals2 [ y ] ) ) ) ) + "<STR_LIT:\n>" ) <EOL> outf . close ( ) <EOL> @ transform ( ( buildExpressionCorrelation , buildExpressionFullCorrelation ) , <EOL> suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def importExpressionCorrelation ( infile , outfile ) : <EOL> table = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ transform ( buildExpressionTracks , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def importExpressionProbesets ( infile , outfile ) : <EOL> table = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ transform ( buildExpressionTracks , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def importExpressionMap ( infile , outfile ) : <EOL> table = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ files_re ( [ x for x in glob . glob ( "<STR_LIT>" ) if not isReplicate ( x [ : - len ( "<STR_LIT>" ) ] ) ] , <EOL> "<STR_LIT>" , <EOL> ( r"<STR_LIT>" , TARGET_PROBESET ) , <EOL> r"<STR_LIT>" ) <EOL> def buildIntervalProbesetAssociations ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> to_cluster = True <EOL> infile , probeset = infiles <EOL> statement = """<STR_LIT>""" <EOL> P . run ( ) <EOL> @ files_re ( buildIntervalProbesetAssociations , <EOL> "<STR_LIT>" , <EOL> r"<STR_LIT>" ) <EOL> def importIntervalProbesetAssociations ( infile , outfile ) : <EOL> tablename = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ transform ( [ "<STR_LIT>" % x for x in TRACKS_MOTIFS ] , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def exportMotifSequences ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> track = infile [ : - len ( "<STR_LIT>" ) ] <EOL> dbhandle = sqlite3 . connect ( PARAMS [ "<STR_LIT>" ] ) <EOL> fasta = IndexedFasta . IndexedFasta ( PARAMS [ "<STR_LIT>" ] ) <EOL> cc = dbhandle . cursor ( ) <EOL> statement = "<STR_LIT>" % track <EOL> cc . execute ( statement ) <EOL> outs = open ( outfile , "<STR_LIT:w>" ) <EOL> for result in cc : <EOL> interval , contig , start , end = result <EOL> id = "<STR_LIT>" % ( track , str ( interval ) , contig , start , end ) <EOL> seq = fasta . getSequence ( contig , "<STR_LIT:+>" , start , end ) <EOL> outs . write ( "<STR_LIT>" % ( id , seq ) ) <EOL> cc . close ( ) <EOL> outs . close ( ) <EOL> @ transform ( [ "<STR_LIT>" % x for x in TRACKS_MOTIFS ] , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def exportMotifControlSequences ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> track = infile [ : - len ( "<STR_LIT>" ) ] <EOL> dbhandle = sqlite3 . connect ( PARAMS [ "<STR_LIT>" ] ) <EOL> fasta = IndexedFasta . IndexedFasta ( PARAMS [ "<STR_LIT>" ] ) <EOL> cc = dbhandle . cursor ( ) <EOL> statement = "<STR_LIT>" % track <EOL> cc . execute ( statement ) <EOL> outs = open ( outfile , "<STR_LIT:w>" ) <EOL> for result in cc : <EOL> interval , contig , xstart , xend = result <EOL> l = xend - xstart <EOL> lcontig = fasta . getLength ( contig ) <EOL> start , end = max ( <NUM_LIT:0> , xstart - l ) , xend - l <EOL> id = "<STR_LIT>" % ( track , str ( interval ) , contig , start , end ) <EOL> seq = fasta . getSequence ( contig , "<STR_LIT:+>" , start , end ) <EOL> outs . write ( "<STR_LIT>" % ( id , seq ) ) <EOL> start , end = xstart + l , min ( lcontig , xend + l ) <EOL> id = "<STR_LIT>" % ( track , str ( interval ) , contig , start , end ) <EOL> seq = fasta . getSequence ( contig , "<STR_LIT:+>" , start , end ) <EOL> outs . write ( "<STR_LIT>" % ( id , seq ) ) <EOL> cc . close ( ) <EOL> outs . close ( ) <EOL> @ merge ( ( exportReferenceAsBed , exportIntervalsAsBed , <EOL> [ "<STR_LIT>" % x for x in TRACKS_REFERENCE ] ) , <EOL> "<STR_LIT>" ) <EOL> def buildOverlap ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> if os . path . exists ( outfile ) : <EOL> os . rename ( outfile , outfile + "<STR_LIT>" ) <EOL> options = "<STR_LIT>" % outfile <EOL> else : <EOL> options = "<STR_LIT>" <EOL> infiles = "<STR_LIT:U+0020>" . join ( IOTools . flatten ( infiles ) ) <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ transform ( buildOverlap , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def importOverlap ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> tablename = "<STR_LIT>" <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ merge ( ( exportUCSCEncodeTracks , <EOL> exportReferenceAsBed , <EOL> [ "<STR_LIT>" % x for x in TRACKS_REFERENCE ] , <EOL> combineConditions , <EOL> combineUnstim , <EOL> combineReplicates ) , <EOL> "<STR_LIT>" ) <EOL> def buildUCSCOverlap ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> if os . path . exists ( outfile ) : <EOL> os . rename ( outfile , outfile + "<STR_LIT>" ) <EOL> options = "<STR_LIT>" % outfile <EOL> else : <EOL> options = "<STR_LIT>" <EOL> to_cluster = True <EOL> infiles = "<STR_LIT:U+0020>" . join ( IOTools . flatten ( infiles ) ) <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ transform ( buildUCSCOverlap , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def importUCSCOverlap ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> tablename = "<STR_LIT>" <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> if REPLICATES : <EOL> @ files ( [ ( [ "<STR_LIT>" % ( x , y ) for y in REPLICATES ] , "<STR_LIT>" % x , x ) for x in TRACKS_EXPERIMENTS ] ) <EOL> def makeReproducibility ( infile , outfile , track ) : <EOL> '''<STR_LIT>''' <EOL> dbhandle = sqlite3 . connect ( PARAMS [ "<STR_LIT>" ] ) <EOL> data = [ ] <EOL> for replicate in REPLICATES : <EOL> cc = dbhandle . cursor ( ) <EOL> statement = "<STR_LIT>" % locals ( ) <EOL> cc . execute ( statement ) <EOL> data . append ( cc . fetchall ( ) ) <EOL> ma = int ( max ( [ x [ <NUM_LIT:3> ] for x in itertools . chain ( * data ) ] ) + <NUM_LIT:1> ) <EOL> nexons1 , nexons2 , nexons_overlapping = [ <NUM_LIT:0> ] * ma , [ <NUM_LIT:0> ] * ma , [ <NUM_LIT:0> ] * ma <EOL> nbases1 , nbases2 , nbases_overlapping = [ <NUM_LIT:0> ] * ma , [ <NUM_LIT:0> ] * ma , [ <NUM_LIT:0> ] * ma <EOL> idx = IndexedGenome . IndexedGenome ( ) <EOL> for contig , start , end , peakval in data [ <NUM_LIT:0> ] : <EOL> peakval = int ( peakval ) <EOL> for x in range ( <NUM_LIT:0> , peakval ) : <EOL> nexons1 [ x ] += <NUM_LIT:1> <EOL> nbases1 [ x ] += end - start <EOL> idx . add ( contig , start , end , peakval ) <EOL> for contig , start , end , peakval in data [ <NUM_LIT:1> ] : <EOL> peakval = int ( peakval ) <EOL> for x in range ( <NUM_LIT:0> , peakval ) : <EOL> nexons2 [ x ] += <NUM_LIT:1> <EOL> nbases2 [ x ] += end - start <EOL> try : <EOL> intervals = list ( idx . get ( contig , start , end ) ) <EOL> except KeyError : <EOL> continue <EOL> if len ( intervals ) == <NUM_LIT:0> : continue <EOL> for other_start , other_end , other_peakval in intervals : <EOL> ovl = min ( end , other_end ) - max ( start , other_start ) <EOL> for x in range ( <NUM_LIT:0> , min ( other_peakval , peakval ) ) : <EOL> nexons_overlapping [ x ] += <NUM_LIT:1> <EOL> nbases_overlapping [ x ] += ovl <EOL> outs = open ( outfile , "<STR_LIT:w>" ) <EOL> outs . write ( "<STR_LIT>" ) <EOL> total_bases = nbases1 [ <NUM_LIT:0> ] + nbases2 [ <NUM_LIT:0> ] - nbases_overlapping [ <NUM_LIT:0> ] <EOL> for x in range ( <NUM_LIT:0> , ma ) : <EOL> bases_union = nbases1 [ x ] + nbases2 [ x ] - nbases_overlapping [ x ] <EOL> exons_union = nexons1 [ x ] + nexons2 [ x ] - nexons_overlapping [ x ] <EOL> outs . write ( "<STR_LIT:\t>" . join ( ( str ( x ) , <EOL> str ( nexons1 [ x ] ) , <EOL> str ( nexons2 [ x ] ) , <EOL> str ( exons_union ) , <EOL> str ( nexons_overlapping [ x ] ) , <EOL> IOTools . prettyPercent ( nbases_overlapping [ x ] , bases_union ) , <EOL> IOTools . prettyPercent ( bases_union , total_bases ) , <EOL> str ( nbases1 [ x ] ) , <EOL> str ( nbases2 [ x ] ) , <EOL> str ( bases_union ) , <EOL> str ( nbases_overlapping [ x ] ) , <EOL> IOTools . prettyPercent ( nbases_overlapping [ x ] , bases_union ) , <EOL> IOTools . prettyPercent ( bases_union , total_bases ) , <EOL> ) ) + "<STR_LIT:\n>" ) <EOL> outs . close ( ) <EOL> @ files_re ( makeReproducibility , "<STR_LIT>" , r"<STR_LIT>" ) <EOL> def importReproducibility ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> tablename = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> @ follows ( importReproducibility ) <EOL> def reproducibility ( ) : pass <EOL> else : <EOL> @ follows ( buildGeneSet ) <EOL> def reproducibility ( ) : pass <EOL> @ files ( [ ( [ "<STR_LIT>" % ( x , y ) for y in REPLICATES ] , "<STR_LIT>" % x ) for x in TRACKS_EXPERIMENTS ] ) <EOL> def makeReadCorrelation ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> to_cluster = True <EOL> job_options = "<STR_LIT>" <EOL> infiles = "<STR_LIT:U+0020>" . join ( infiles ) <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ merge ( makeReadCorrelation , "<STR_LIT>" ) <EOL> def makeReadCorrelationTable ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> correlation_cutoff = <NUM_LIT:5> <EOL> outf = open ( outfile , "<STR_LIT:w>" ) <EOL> outf . write ( "<STR_LIT>" ) <EOL> def selector ( infile ) : <EOL> correlation_cutoff = <NUM_LIT:5> <EOL> for line in infile : <EOL> try : <EOL> data = map ( int , line [ : - <NUM_LIT:1> ] . split ( "<STR_LIT:\t>" ) [ <NUM_LIT:2> : ] ) <EOL> except ValueError : <EOL> continue <EOL> for x in range ( len ( data ) ) : <EOL> if data [ x ] < correlation_cutoff : break <EOL> else : <EOL> yield data <EOL> for infile in infiles : <EOL> mat = numpy . array ( [ x for x in selector ( gzip . open ( infile , "<STR_LIT:r>" ) ) ] ) <EOL> corr = numpy . corrcoef ( mat [ : , <NUM_LIT:0> ] , mat [ : , <NUM_LIT:1> ] ) <EOL> outf . write ( "<STR_LIT>" % ( <EOL> infile , <EOL> correlation_cutoff , <EOL> len ( mat ) , <EOL> corr [ <NUM_LIT:0> , <NUM_LIT:1> ] ) ) <EOL> outf . flush ( ) <EOL> outf . close ( ) <EOL> @ follows ( makeMerged ) <EOL> @ files_re ( [ "<STR_LIT>" % x for x in TRACKS_CORRELATION ] , <EOL> combine ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def makePeakvalCorrelation ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . makeIntervalCorrelation ( infiles , outfile , "<STR_LIT>" ) <EOL> @ follows ( makeMerged ) <EOL> @ files_re ( [ "<STR_LIT>" % x for x in TRACKS_CORRELATION ] , <EOL> combine ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def makeAvgvalCorrelation ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . makeIntervalCorrelation ( infiles , outfile , "<STR_LIT>" ) <EOL> @ follows ( makeMerged ) <EOL> @ files_re ( [ "<STR_LIT>" % x for x in TRACKS_CORRELATION ] , <EOL> combine ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def makeLengthCorrelation ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineChipseq . makeIntervalCorrelation ( infiles , outfile , "<STR_LIT>" ) <EOL> @ transform ( ( makePeakvalCorrelation , makeAvgvalCorrelation , makeLengthCorrelation ) , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def importCorrelation ( infile , outfile ) : <EOL> tablename = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> @ files_re ( "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> r"<STR_LIT>" ) <EOL> def makeMotifs ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> tmpdir = tempfile . mkdtemp ( ) <EOL> tmpname = os . path . join ( tmpdir , "<STR_LIT>" ) <EOL> tmpfile = open ( tmpname , "<STR_LIT:w>" ) <EOL> for line in open ( infile , "<STR_LIT:r>" ) : <EOL> tmpfile . write ( re . sub ( "<STR_LIT>" , "<STR_LIT:_>" , line ) ) <EOL> tmpfile . close ( ) <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> shutil . rmtree ( tmpdir ) <EOL> @ transform ( exportMotifSequences , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def runMEME ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> to_cluster = True <EOL> job_queue = "<STR_LIT>" <EOL> job_options = "<STR_LIT>" <EOL> target_path = os . path . join ( os . path . abspath ( PARAMS [ "<STR_LIT>" ] ) , "<STR_LIT>" , outfile ) <EOL> dbhandle = sqlite3 . connect ( PARAMS [ "<STR_LIT>" ] ) <EOL> fasta = IndexedFasta . IndexedFasta ( PARAMS [ "<STR_LIT>" ] ) <EOL> track = infile [ : - len ( "<STR_LIT>" ) ] <EOL> cc = dbhandle . cursor ( ) <EOL> statement = "<STR_LIT>" % track <EOL> cc . execute ( statement ) <EOL> data = cc . fetchall ( ) <EOL> cc . close ( ) <EOL> cutoff = len ( data ) // <NUM_LIT:10> <EOL> maxsize = int ( PARAMS [ "<STR_LIT>" ] ) <EOL> E . info ( "<STR_LIT>" % ( track , cutoff ) ) <EOL> fasta = IndexedFasta . IndexedFasta ( PARAMS [ "<STR_LIT>" ] ) <EOL> tmpdir = tempfile . mkdtemp ( dir = "<STR_LIT:.>" ) <EOL> tmpfasta = os . path . join ( tmpdir , "<STR_LIT>" ) <EOL> outs = open ( tmpfasta , "<STR_LIT:w>" ) <EOL> masker = Masker . MaskerDustMasker ( ) <EOL> hw = int ( PARAMS [ "<STR_LIT>" ] ) <EOL> current_size , nseq = <NUM_LIT:0> , <NUM_LIT:0> <EOL> for peakcenter , id , contig in data [ : cutoff ] : <EOL> start , end = peakcenter - hw , peakcenter + hw <EOL> id = "<STR_LIT>" % ( track , str ( id ) , contig , start , end ) <EOL> seq = fasta . getSequence ( contig , "<STR_LIT:+>" , start , end ) <EOL> if PARAMS [ "<STR_LIT>" ] == "<STR_LIT>" : <EOL> masked_seq = seq <EOL> elif PARAMS [ "<STR_LIT>" ] == "<STR_LIT>" : <EOL> masked_seq = masker ( seq . upper ( ) ) <EOL> masked_seq = re . sub ( "<STR_LIT>" , "<STR_LIT:N>" , masked_seq ) <EOL> current_size += len ( masked_seq ) <EOL> if current_size >= maxsize : <EOL> E . info ( "<STR_LIT>" % ( track , maxsize , nseq ) ) <EOL> break <EOL> nseq += <NUM_LIT:1> <EOL> outs . write ( "<STR_LIT>" % ( id , masked_seq ) ) <EOL> outs . close ( ) <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> try : <EOL> os . makedirs ( os . path . dirname ( target_path ) ) <EOL> except OSError : <EOL> pass <EOL> if os . path . exists ( target_path ) : shutil . rmtree ( target_path ) <EOL> shutil . move ( tmpdir , target_path ) <EOL> shutil . copyfile ( os . path . join ( target_path , "<STR_LIT>" ) , outfile ) <EOL> def writeSequencesForIntervals ( track , filename , <EOL> full = False , <EOL> halfwidth = None , <EOL> maxsize = None , <EOL> proportion = None , <EOL> masker = None , <EOL> offset = <NUM_LIT:0> , <EOL> shuffled = False ) : <EOL> '''<STR_LIT>''' <EOL> dbhandle = sqlite3 . connect ( PARAMS [ "<STR_LIT>" ] ) <EOL> fasta = IndexedFasta . IndexedFasta ( PARAMS [ "<STR_LIT>" ] ) <EOL> cc = dbhandle . cursor ( ) <EOL> if full : <EOL> statement = '''<STR_LIT>''' % locals ( ) <EOL> elif halfwidth : <EOL> statement = '''<STR_LIT>''' % locals ( ) <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> cc . execute ( statement ) <EOL> data = cc . fetchall ( ) <EOL> cc . close ( ) <EOL> if proportion : <EOL> cutoff = int ( len ( data ) * proportion ) <EOL> else : <EOL> cutoff = len ( data ) <EOL> E . info ( "<STR_LIT>" % ( track , cutoff ) ) <EOL> E . info ( "<STR_LIT>" % ( track , masker ) ) <EOL> fasta = IndexedFasta . IndexedFasta ( PARAMS [ "<STR_LIT>" ] ) <EOL> masker_object = Masker . MaskerDustMasker ( ) <EOL> sequences = [ ] <EOL> current_size , nseq = <NUM_LIT:0> , <NUM_LIT:0> <EOL> new_data = [ ] <EOL> for start , end , id , contig in data [ : cutoff ] : <EOL> lcontig = fasta . getLength ( contig ) <EOL> start , end = max ( <NUM_LIT:0> , start + offset ) , min ( end + offset , lcontig ) <EOL> if start >= end : <EOL> E . info ( "<STR_LIT>" % ( track , id , start , end , offset ) ) <EOL> continue <EOL> seq = fasta . getSequence ( contig , "<STR_LIT:+>" , start , end ) <EOL> sequences . append ( seq ) <EOL> new_data . append ( ( start , end , id , contig ) ) <EOL> current_size += len ( seq ) <EOL> if maxsize and current_size >= maxsize : <EOL> E . info ( "<STR_LIT>" % ( track , maxsize , nseq , <EOL> len ( data ) - nseq ) ) <EOL> break <EOL> nseq += <NUM_LIT:1> <EOL> data = new_data <EOL> def maskSequences ( sequences , masker ) : <EOL> if masker == "<STR_LIT>" : <EOL> masked_seq = sequences <EOL> elif masker == "<STR_LIT>" : <EOL> masked_seq = [ masker_object ( x . upper ( ) ) for x in sequences ] <EOL> else : <EOL> masked_seq = [ x . upper ( ) for x in sequences ] <EOL> masked_seq = [ re . sub ( "<STR_LIT>" , "<STR_LIT:N>" , x ) for x in masked_seq ] <EOL> return masked_seq <EOL> if shuffled : <EOL> sequences = [ list ( x ) for x in sequences ] <EOL> for sequence in sequences : random . shuffle ( sequence ) <EOL> sequences = maskSequences ( [ "<STR_LIT>" . join ( x ) for x in sequences ] , masker ) <EOL> outs = open ( filename , "<STR_LIT:w>" ) <EOL> for sequence , d in zip ( maskSequences ( sequences , masker ) , data ) : <EOL> start , end , id , contig = d <EOL> id = "<STR_LIT>" % ( track , str ( id ) , contig , start , end ) <EOL> outs . write ( "<STR_LIT>" % ( id , sequence ) ) <EOL> outs . close ( ) <EOL> return len ( sequences ) <EOL> @ transform ( exportMotifSequences , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def runGLAM2 ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> to_cluster = True <EOL> job_queue = "<STR_LIT>" <EOL> target_path = os . path . join ( os . path . abspath ( PARAMS [ "<STR_LIT>" ] ) , "<STR_LIT>" , outfile ) <EOL> track = infile [ : - len ( "<STR_LIT>" ) ] <EOL> tmpdir = tempfile . mkdtemp ( ) <EOL> tmpfasta = os . path . join ( tmpdir , "<STR_LIT>" ) <EOL> nseq = writeSequencesForIntervals ( track , tmpfasta , <EOL> full = False , <EOL> halfwidth = int ( PARAMS [ "<STR_LIT>" ] ) , <EOL> maxsize = int ( PARAMS [ "<STR_LIT>" ] ) , <EOL> proportion = <NUM_LIT:0.1> ) <EOL> min_sequences = int ( nseq / <NUM_LIT> ) <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> try : <EOL> os . makedirs ( os . path . dirname ( target_path ) ) <EOL> except OSError : <EOL> pass <EOL> if os . path . exists ( target_path ) : shutil . rmtree ( target_path ) <EOL> shutil . move ( tmpdir , target_path ) <EOL> shutil . copyfile ( os . path . join ( target_path , "<STR_LIT>" ) , outfile ) <EOL> if PARAMS [ "<STR_LIT>" ] != "<STR_LIT>" : <EOL> @ follows ( makeMotifs ) <EOL> @ transform ( runMEME , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def runTomTom ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> to_cluster = True <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ transform ( runTomTom , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def importTomTom ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> tmpfile = P . getTempFile ( ) <EOL> tmpfile . write ( "<STR_LIT:\t>" . join ( ( "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" ) ) + "<STR_LIT:\n>" ) <EOL> for line in open ( infile , "<STR_LIT:r>" ) : <EOL> if line . startswith ( "<STR_LIT>" ) : continue <EOL> ( query_id , target_id , <EOL> optimal_offset , pvalue , qvalue , overlap , query_consensus , <EOL> target_consensus , orientation ) = line [ : - <NUM_LIT:1> ] . split ( "<STR_LIT:\t>" ) <EOL> tmpfile . write ( line ) <EOL> tmpfile . close ( ) <EOL> tmpname = tmpfile . name <EOL> tablename = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> os . unlink ( tmpname ) <EOL> @ transform ( runTomTom , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def filterMotifs ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> infile_meme = infile [ : - len ( "<STR_LIT>" ) ] + "<STR_LIT>" <EOL> selected = [ ] <EOL> max_pvalue = float ( PARAMS [ "<STR_LIT>" ] ) <EOL> for line in open ( infile , "<STR_LIT:r>" ) : <EOL> if line . startswith ( "<STR_LIT>" ) : continue <EOL> ( query_id , target_id , <EOL> optimal_offset , pvalue , qvalue , overlap , query_consensus , <EOL> target_consensus , orientation ) = line [ : - <NUM_LIT:1> ] . split ( "<STR_LIT:\t>" ) <EOL> if float ( pvalue ) <= max_pvalue : <EOL> selected . append ( target_id ) <EOL> E . info ( "<STR_LIT>" % ( infile , len ( selected ) ) ) <EOL> PipelineMotifs . filterMotifsFromMEME ( infile_meme , outfile , selected ) <EOL> else : <EOL> @ follows ( runMEME ) <EOL> def runTomTom ( ) : pass <EOL> @ follows ( runTomTom ) <EOL> def importTomTom ( ) : pass <EOL> @ transform ( runMEME , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def filterMotifs ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PipelineMotifs . filterMotifsFromMEME ( infile , outfile , [ "<STR_LIT:1>" ] ) <EOL> @ follows ( makeMotifs , filterMotifs ) <EOL> @ files_re ( ( exportMotifSequences , exportMotifControlSequences ) , <EOL> "<STR_LIT>" , <EOL> [ r"<STR_LIT>" , r"<STR_LIT>" , glob . glob ( "<STR_LIT>" ) ] , <EOL> r"<STR_LIT>" ) <EOL> def runMAST ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> to_cluster = True <EOL> job_queue = "<STR_LIT>" <EOL> job_options = "<STR_LIT>" <EOL> controlfile , dbfile , motiffiles = infiles <EOL> controlfile = dbfile [ : - len ( "<STR_LIT>" ) ] + "<STR_LIT>" <EOL> if not os . path . exists ( controlfile ) : <EOL> raise P . PipelineError ( "<STR_LIT>" % ( controlfile , dbfile ) ) <EOL> if os . path . exists ( outfile ) : os . remove ( outfile ) <EOL> for motiffile in motiffiles : <EOL> if IOTools . isEmpty ( motiffile ) : <EOL> E . info ( "<STR_LIT>" % motiffile ) <EOL> continue <EOL> of = gzip . open ( outfile , "<STR_LIT:a>" ) <EOL> motif , x = os . path . splitext ( motiffile ) <EOL> of . write ( "<STR_LIT>" % motif ) <EOL> of . close ( ) <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ transform ( exportMotifSequences , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def runBioProspector ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> to_cluster = True <EOL> job_options = "<STR_LIT>" <EOL> tmpfasta = P . getTempFilename ( "<STR_LIT:.>" ) <EOL> track = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> nseq = writeSequencesForIntervals ( track , <EOL> tmpfasta , <EOL> full = True , <EOL> masker = "<STR_LIT>" , <EOL> proportion = <NUM_LIT> ) <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> os . unlink ( tmpfasta ) <EOL> @ transform ( runBioProspector , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def importBioProspector ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> tablename = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> target_path = os . path . join ( os . path . abspath ( PARAMS [ "<STR_LIT>" ] ) , "<STR_LIT>" ) <EOL> try : <EOL> os . makedirs ( target_path ) <EOL> except OSError : <EOL> pass <EOL> track = infile [ : - len ( "<STR_LIT>" ) ] <EOL> results = Bioprospector . parse ( open ( infile , "<STR_LIT:r>" ) ) <EOL> tmpfile = P . getTempFile ( ) <EOL> tmpfile . write ( "<STR_LIT>" ) <EOL> for x , motifs in enumerate ( results ) : <EOL> outname = os . path . join ( target_path , "<STR_LIT>" % ( track , x ) ) <EOL> Bioprospector . build_logo ( [ y . sequence for y in motifs . matches ] , <EOL> outname ) <EOL> for match in motifs . matches : <EOL> distance = abs ( match . start + match . width1 - ( match . end - match . width2 ) ) <EOL> if match . strand in ( "<STR_LIT>" , "<STR_LIT>" ) : <EOL> arrangement = "<STR_LIT>" <EOL> elif match . strand in ( "<STR_LIT>" , "<STR_LIT>" ) : <EOL> arrangement = "<STR_LIT>" <EOL> else : <EOL> arrangement = "<STR_LIT>" <EOL> distance = <NUM_LIT:0> <EOL> arrangement += "<STR_LIT>" % distance <EOL> strand = match . strand [ <NUM_LIT:0> ] <EOL> id = re . sub ( "<STR_LIT>" , "<STR_LIT>" , match . id ) <EOL> tmpfile . write ( "<STR_LIT>" % ( id , <EOL> x , <EOL> match . start , <EOL> match . end , <EOL> strand , <EOL> arrangement ) ) <EOL> tmpfile . close ( ) <EOL> tmpfilename = tmpfile . name <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> @ files_re ( ( exportMotifSequences , exportMotifControlSequences ) , <EOL> "<STR_LIT>" , <EOL> [ r"<STR_LIT>" , r"<STR_LIT>" ] , <EOL> r"<STR_LIT>" ) <EOL> def runRegexMotifSearch ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> motif = "<STR_LIT>" <EOL> reverse_motif = "<STR_LIT>" <EOL> controlfile , dbfile = infiles <EOL> controlfile = dbfile [ : - len ( "<STR_LIT>" ) ] + "<STR_LIT>" <EOL> if not os . path . exists ( controlfile ) : <EOL> raise P . PipelineError ( "<STR_LIT>" % ( controlfile , dbfile ) ) <EOL> motifs = [ ] <EOL> for x in range ( <NUM_LIT:0> , <NUM_LIT:15> ) : <EOL> motifs . append ( ( "<STR_LIT>" % x , re . compile ( motif + "<STR_LIT:.>" * x + motif , re . IGNORECASE ) ) ) <EOL> for x in range ( <NUM_LIT:0> , <NUM_LIT:15> ) : <EOL> motifs . append ( ( "<STR_LIT>" % x , re . compile ( motif + "<STR_LIT:.>" * x + reverse_motif , re . IGNORECASE ) ) ) <EOL> db_positions = Motifs . countMotifs ( open ( dbfile , "<STR_LIT:r>" ) , motifs ) <EOL> control_positions = Motifs . countMotifs ( open ( controlfile , "<STR_LIT:r>" ) , motifs ) <EOL> db_counts , control_counts = Motifs . getCounts ( db_positions ) , Motifs . getCounts ( control_positions ) <EOL> db_seqcounts , control_seqcounts = Motifs . getOccurances ( db_positions ) , Motifs . getCounts ( control_positions ) <EOL> ndb , ncontrol = len ( db_positions ) , len ( control_positions ) <EOL> outf = open ( outfile , "<STR_LIT:w>" ) <EOL> outf . write ( "<STR_LIT>" ) <EOL> for motif , pattern in motifs : <EOL> try : <EOL> fold = float ( db_seqcounts [ motif ] ) * ncontrol / ( ndb * control_seqcounts [ motif ] ) <EOL> except ZeroDivisionError : <EOL> fold = <NUM_LIT:0> <EOL> outf . write ( "<STR_LIT>" % ( motif , <EOL> db_counts [ motif ] , <EOL> control_counts [ motif ] , <EOL> db_seqcounts [ motif ] , <EOL> IOTools . prettyPercent ( db_seqcounts [ motif ] , ndb ) , <EOL> control_seqcounts [ motif ] , <EOL> IOTools . prettyPercent ( control_seqcounts [ motif ] , ncontrol ) , <EOL> fold ) ) <EOL> def _runNubiscanMotifSearch ( infile , outfile , motif , mode = "<STR_LIT>" ) : <EOL> '''<STR_LIT>''' <EOL> to_cluster = True <EOL> job_options = "<STR_LIT>" <EOL> track = re . sub ( "<STR_LIT>" , "<STR_LIT>" , outfile ) <EOL> tmpfasta = P . getTempFilename ( "<STR_LIT:.>" ) <EOL> if mode == "<STR_LIT>" : <EOL> nseq = writeSequencesForIntervals ( track , <EOL> tmpfasta , <EOL> full = False , <EOL> halfwidth = <NUM_LIT:100> , <EOL> shuffled = True ) <EOL> elif mode == "<STR_LIT>" : <EOL> nseq = writeSequencesForIntervals ( track , <EOL> tmpfasta , <EOL> full = False , <EOL> halfwidth = <NUM_LIT:100> , <EOL> offset = - <NUM_LIT> ) <EOL> elif mode == "<STR_LIT>" : <EOL> nseq = writeSequencesForIntervals ( track , <EOL> tmpfasta , <EOL> full = False , <EOL> halfwidth = <NUM_LIT:100> ) <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" % mode ) <EOL> arrangements = "<STR_LIT:U+002C>" . join ( PARAMS [ "<STR_LIT>" ] ) <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> os . unlink ( tmpfasta ) <EOL> @ transform ( exportMotifSequences , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def runNubiscanVDR ( infile , outfile ) : <EOL> _runNubiscanMotifSearch ( infile , outfile , motif = "<STR_LIT>" , mode = "<STR_LIT>" ) <EOL> @ transform ( exportMotifSequences , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def runNubiscanVDRShuffled ( infile , outfile ) : <EOL> _runNubiscanMotifSearch ( infile , outfile , motif = "<STR_LIT>" , mode = "<STR_LIT>" ) <EOL> @ transform ( exportMotifSequences , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def runNubiscanVDRShifted ( infile , outfile ) : <EOL> _runNubiscanMotifSearch ( infile , outfile , motif = "<STR_LIT>" , mode = "<STR_LIT>" ) <EOL> @ transform ( exportMotifSequences , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def runNubiscanNR ( infile , outfile ) : <EOL> _runNubiscanMotifSearch ( infile , outfile , motif = "<STR_LIT>" , mode = "<STR_LIT>" ) <EOL> @ transform ( exportMotifSequences , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def runNubiscanNRShuffled ( infile , outfile ) : <EOL> _runNubiscanMotifSearch ( infile , outfile , motif = "<STR_LIT>" , mode = "<STR_LIT>" ) <EOL> @ transform ( exportMotifSequences , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def runNubiscanNRShifted ( infile , outfile ) : <EOL> _runNubiscanMotifSearch ( infile , outfile , motif = "<STR_LIT>" , mode = "<STR_LIT>" ) <EOL> @ transform ( ( runNubiscanVDR , runNubiscanNR ) , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def importNubiscan ( infile , outfile ) : <EOL> tablename = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> track = tablename [ : - len ( "<STR_LIT>" ) ] <EOL> tmpfile = P . getTempFile ( ) <EOL> first = True <EOL> for line in fileinput . input ( glob . glob ( "<STR_LIT>" % ( track ) ) ) : <EOL> if fileinput . filename ( ) . endswith ( "<STR_LIT>" ) : <EOL> fileinput . nextfile ( ) <EOL> continue <EOL> if line . startswith ( "<STR_LIT:#>" ) : continue <EOL> if re . match ( "<STR_LIT>" , line ) : <EOL> base , motif = os . path . splitext ( fileinput . filename ( ) ) <EOL> if motif == "<STR_LIT>" : <EOL> base , motif = os . path . splitext ( base ) <EOL> motif += "<STR_LIT>" <EOL> elif motif == "<STR_LIT>" : <EOL> base , motif = os . path . splitext ( base ) <EOL> motif += "<STR_LIT>" <EOL> motif = motif [ <NUM_LIT:1> : ] <EOL> if first : <EOL> tmpfile . write ( "<STR_LIT>" % line ) <EOL> first = False <EOL> continue <EOL> line = re . sub ( "<STR_LIT>" , "<STR_LIT>" , line ) <EOL> data = line . split ( "<STR_LIT:\t>" ) <EOL> data [ <NUM_LIT:0> ] = re . sub ( "<STR_LIT:U+0020>" , "<STR_LIT>" , data [ <NUM_LIT:0> ] ) <EOL> line = "<STR_LIT:\t>" . join ( data ) <EOL> tmpfile . write ( "<STR_LIT>" % ( motif , line ) ) <EOL> tmpfile . close ( ) <EOL> tmpfilename = tmpfile . name <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> os . unlink ( tmpfilename ) <EOL> @ transform ( runMAST , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def importMAST ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> tablename = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> tmpfile = tempfile . NamedTemporaryFile ( delete = False ) <EOL> tmpfile . write ( MAST . Match ( ) . header + "<STR_LIT>" "<STR_LIT>" "<STR_LIT>" "<STR_LIT>" + "<STR_LIT:\n>" ) <EOL> lines = gzip . open ( infile ) . readlines ( ) <EOL> chunks = [ x for x in range ( len ( lines ) ) if lines [ x ] . startswith ( "<STR_LIT>" ) ] <EOL> chunks . append ( len ( lines ) ) <EOL> for chunk in range ( len ( chunks ) - <NUM_LIT:1> ) : <EOL> tmpfile2 = tempfile . NamedTemporaryFile ( delete = False ) <EOL> try : <EOL> motif = re . match ( "<STR_LIT>" , lines [ chunks [ chunk ] ] ) . groups ( ) [ <NUM_LIT:0> ] <EOL> except AttributeError : <EOL> raise P . PipelineError ( "<STR_LIT>" % lines [ chunks [ chunk ] ] ) <EOL> tmpfile2 . write ( "<STR_LIT>" . join ( lines [ chunks [ chunk ] + <NUM_LIT:1> : chunks [ chunk + <NUM_LIT:1> ] ] ) ) <EOL> tmpfile2 . close ( ) <EOL> mast = MAST . parse ( open ( tmpfile2 . name , "<STR_LIT:r>" ) ) <EOL> os . unlink ( tmpfile2 . name ) <EOL> full_matches = [ ] <EOL> controls = collections . defaultdict ( dict ) <EOL> for match in mast . matches : <EOL> m = match . id . split ( "<STR_LIT:_>" ) <EOL> if len ( m ) == <NUM_LIT:2> : <EOL> full_matches . append ( match ) <EOL> continue <EOL> track , id , pos = m <EOL> controls [ id ] [ pos ] = ( match . evalue , match . pvalue , match . nmotifs , match . length , match . start , match . end ) <EOL> for match in full_matches : <EOL> match . id = match . id . split ( "<STR_LIT:_>" ) [ <NUM_LIT:1> ] <EOL> contig , start , end = re . match ( "<STR_LIT>" , match . description ) . groups ( ) <EOL> if match . nmotifs > <NUM_LIT:0> : <EOL> start , end = int ( start ) , int ( end ) <EOL> match . start += start <EOL> match . end += start <EOL> match . positions = [ x + start for x in match . positions ] <EOL> id = match . id <EOL> if id not in controls : <EOL> P . warn ( "<STR_LIT>" % id ) <EOL> if "<STR_LIT:l>" not in controls [ id ] : controls [ id ] [ "<STR_LIT:l>" ] = ( float ( PARAMS [ "<STR_LIT>" ] ) , <NUM_LIT:1> , <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> ) <EOL> if "<STR_LIT:r>" not in controls [ id ] : controls [ id ] [ "<STR_LIT:r>" ] = ( float ( PARAMS [ "<STR_LIT>" ] ) , <NUM_LIT:1> , <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> ) <EOL> min_evalue = min ( controls [ id ] [ "<STR_LIT:l>" ] [ <NUM_LIT:0> ] , controls [ id ] [ "<STR_LIT:r>" ] [ <NUM_LIT:0> ] ) <EOL> min_pvalue = min ( controls [ id ] [ "<STR_LIT:l>" ] [ <NUM_LIT:1> ] , controls [ id ] [ "<STR_LIT:r>" ] [ <NUM_LIT:1> ] ) <EOL> max_nmatches = max ( controls [ id ] [ "<STR_LIT:l>" ] [ <NUM_LIT:2> ] , controls [ id ] [ "<STR_LIT:r>" ] [ <NUM_LIT:2> ] ) <EOL> tmpfile . write ( str ( match ) + "<STR_LIT>" % ( motif , contig , <EOL> "<STR_LIT:\t>" . join ( map ( str , controls [ id ] [ "<STR_LIT:l>" ] ) ) , <EOL> "<STR_LIT:\t>" . join ( map ( str , controls [ id ] [ "<STR_LIT:r>" ] ) ) , <EOL> str ( min_evalue ) , <EOL> str ( min_pvalue ) , <EOL> str ( max_nmatches ) , <EOL> ) + "<STR_LIT:\n>" ) <EOL> tmpfile . close ( ) <EOL> tmpfilename = tmpfile . name <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> os . unlink ( tmpfile . name ) <EOL> @ follows ( makeMotifs , runGLAM2 ) <EOL> @ files_re ( ( exportMotifSequences , exportMotifControlSequences ) , <EOL> "<STR_LIT>" , <EOL> [ r"<STR_LIT>" , r"<STR_LIT>" , glob . glob ( "<STR_LIT>" ) ] , <EOL> r"<STR_LIT>" ) <EOL> def runGLAM2SCAN ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> to_cluster = True <EOL> job_queue = "<STR_LIT>" <EOL> job_options = "<STR_LIT>" <EOL> controlfile , dbfile , motiffiles = infiles <EOL> controlfile = dbfile [ : - len ( "<STR_LIT>" ) ] + "<STR_LIT>" <EOL> if not os . path . exists ( controlfile ) : <EOL> raise P . PipelineError ( "<STR_LIT>" % ( controlfile , dbfile ) ) <EOL> if os . path . exists ( outfile ) : os . remove ( outfile ) <EOL> for motiffile in motiffiles : <EOL> of = open ( outfile , "<STR_LIT:a>" ) <EOL> motif , x = os . path . splitext ( motiffile ) <EOL> of . write ( "<STR_LIT>" % motif ) <EOL> of . close ( ) <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> @ transform ( runGLAM2SCAN , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def importGLAM2SCAN ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> tablename = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> tmpfile = tempfile . NamedTemporaryFile ( delete = False ) <EOL> tmpfile . write ( "<STR_LIT>" ) <EOL> lines = open ( infile ) . readlines ( ) <EOL> chunks = [ x for x in range ( len ( lines ) ) if lines [ x ] . startswith ( "<STR_LIT>" ) ] <EOL> chunks . append ( len ( lines ) ) <EOL> for chunk in range ( len ( chunks ) - <NUM_LIT:1> ) : <EOL> try : <EOL> motif = re . match ( "<STR_LIT>" , lines [ chunks [ chunk ] ] ) . groups ( ) [ <NUM_LIT:0> ] <EOL> except AttributeError : <EOL> raise P . PipelineError ( "<STR_LIT>" % lines [ chunks [ chunk ] ] ) <EOL> if chunks [ chunk ] + <NUM_LIT:1> == chunks [ chunk + <NUM_LIT:1> ] : <EOL> E . warn ( "<STR_LIT>" % motif ) <EOL> continue <EOL> tmpfile2 = tempfile . NamedTemporaryFile ( delete = False ) <EOL> tmpfile2 . write ( "<STR_LIT>" . join ( lines [ chunks [ chunk ] + <NUM_LIT:1> : chunks [ chunk + <NUM_LIT:1> ] ] ) ) <EOL> tmpfile2 . close ( ) <EOL> glam = Glam2Scan . parse ( open ( tmpfile2 . name , "<STR_LIT:r>" ) ) <EOL> os . unlink ( tmpfile2 . name ) <EOL> full_matches = collections . defaultdict ( list ) <EOL> controls = collections . defaultdict ( list ) <EOL> for match in glam . matches : <EOL> m = match . id . split ( "<STR_LIT:_>" ) <EOL> track , id = m [ : <NUM_LIT:2> ] <EOL> if len ( m ) == <NUM_LIT:2> : <EOL> full_matches [ id ] . append ( match ) <EOL> else : <EOL> controls [ id ] . append ( match . score ) <EOL> for id , matches in full_matches . iteritems ( ) : <EOL> nmatches = len ( matches ) <EOL> scores = [ x . score for x in matches ] <EOL> score = max ( scores ) <EOL> contig = "<STR_LIT>" <EOL> if id not in controls : <EOL> P . warn ( "<STR_LIT>" % id ) <EOL> c = controls [ id ] <EOL> if len ( c ) == <NUM_LIT:0> : mmax = "<STR_LIT>" <EOL> else : mmax = max ( c ) <EOL> tmpfile . write ( "<STR_LIT:\t>" . join ( map ( str , <EOL> ( motif , id , <EOL> nmatches , <EOL> score , <EOL> "<STR_LIT:U+002C>" . join ( map ( str , scores ) ) , <EOL> len ( c ) , <EOL> mmax ) ) ) + "<STR_LIT:\n>" ) <EOL> tmpfile . close ( ) <EOL> tmpfilename = tmpfile . name <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> os . unlink ( tmpfile . name ) <EOL> @ files ( [ ( "<STR_LIT>" % x , "<STR_LIT>" % x ) for x in TRACKS_ANNOTATE ] ) <EOL> def annotateIntervals ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> to_cluster = True <EOL> statement = """<STR_LIT>""" <EOL> P . run ( ) <EOL> def straightImport ( infile , outfile , options = "<STR_LIT>" ) : <EOL> '''<STR_LIT>''' <EOL> tablename = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ follows ( annotateIntervals ) <EOL> @ files_re ( "<STR_LIT>" , "<STR_LIT>" , r"<STR_LIT>" ) <EOL> def importAnnotations ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> straightImport ( infile , outfile ) <EOL> @ files ( [ ( "<STR_LIT>" % x , "<STR_LIT>" % x , "<STR_LIT>" ) for x in TRACKS_ANNOTATE ] ) <EOL> def annotateTracks ( infile , outfile , trackfile ) : <EOL> '''<STR_LIT>''' <EOL> statement = """<STR_LIT>""" <EOL> P . run ( ) <EOL> @ transform ( annotateTracks , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def importTracks ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> straightImport ( infile , outfile ) <EOL> @ follows ( buildGeneRegions ) <EOL> @ files ( [ ( "<STR_LIT>" % x , "<STR_LIT>" % x ) for x in TRACKS_ANNOTATE ] ) <EOL> def annotateTSS ( infile , outfile ) : <EOL> to_cluster = True <EOL> statement = """<STR_LIT>""" <EOL> P . run ( ) <EOL> @ files ( [ ( "<STR_LIT>" % x , "<STR_LIT>" % x ) for x in TRACKS_ANNOTATE ] ) <EOL> def annotateRepeats ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> to_cluster = True <EOL> statement = """<STR_LIT>""" <EOL> P . run ( ) <EOL> @ files_re ( annotateTSS , "<STR_LIT>" , r"<STR_LIT>" ) <EOL> def importTSS ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> straightImport ( infile , outfile , options = "<STR_LIT>" ) <EOL> @ transform ( annotateRepeats , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def importRepeats ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> straightImport ( infile , outfile ) <EOL> @ transform ( importExpressionTracks , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def buildExpressionDifferencesTTest ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> nskipped = <NUM_LIT:0> <EOL> track = infile [ : - len ( "<STR_LIT>" ) ] <EOL> outs = open ( outfile , "<STR_LIT:w>" ) <EOL> outs . write ( "<STR_LIT>" ) <EOL> try : <EOL> control , cluster_ids , treatments , controls = PExpression . getExpressionMeasurements ( track ) <EOL> except P . PipelineError , msg : <EOL> E . warn ( "<STR_LIT>" % msg ) <EOL> outs . close ( ) <EOL> return <EOL> E . info ( "<STR_LIT>" % ( track , control , <EOL> len ( cluster_ids ) , <EOL> len ( treatments ) , <EOL> len ( controls ) ) ) <EOL> result , nskipped = Expression . WelchsTTest ( ) ( cluster_ids , treatments , controls ) <EOL> for s in result : <EOL> fold = math . pow ( <NUM_LIT:2> , s . mMean1 - s . mMean2 ) <EOL> outs . write ( "<STR_LIT>" % ( s . mProbeset , <EOL> s . mMean1 , <EOL> s . mSampleVariance1 , <EOL> s . mMean2 , <EOL> s . mSampleVariance2 , <EOL> s . mDegreesFreedom , <EOL> s . mPValue , <EOL> s . mQValue , <EOL> s . mDifference , <EOL> s . mDifferenceLower , <EOL> s . mDifferenceUpper , <EOL> fold , <EOL> math . pow ( <NUM_LIT:2> , s . mDifferenceLower ) , <EOL> math . pow ( <NUM_LIT:2> , s . mDifferenceUpper ) , <EOL> ) ) <EOL> outs . close ( ) <EOL> @ transform ( importExpressionTracks , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def buildExpressionDifferencesSAM ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> nskipped = <NUM_LIT:0> <EOL> track = infile [ : - len ( "<STR_LIT>" ) ] <EOL> target_path = os . path . join ( os . path . abspath ( PARAMS [ "<STR_LIT>" ] ) , "<STR_LIT>" ) <EOL> if not os . path . exists ( target_path ) : <EOL> try : <EOL> os . makedirs ( target_path ) <EOL> except OSError : <EOL> pass <EOL> outs = open ( outfile , "<STR_LIT:w>" ) <EOL> outs . write ( "<STR_LIT>" ) <EOL> try : <EOL> control , cluster_ids , treatments , controls = PExpression . getExpressionMeasurements ( track ) <EOL> except P . PipelineError , msg : <EOL> E . warn ( "<STR_LIT>" % msg ) <EOL> outs . close ( ) <EOL> return <EOL> E . info ( "<STR_LIT>" % ( track , control , <EOL> len ( cluster_ids ) , <EOL> len ( treatments ) , <EOL> len ( controls ) ) ) <EOL> genes , summary , cutoffs = Expression . SAM ( ) ( cluster_ids , treatments , controls , <EOL> pattern = os . path . join ( target_path , outfile + "<STR_LIT:%s>" ) , <EOL> fdr = float ( PARAMS [ "<STR_LIT>" ] ) , <EOL> ngenes = float ( PARAMS [ "<STR_LIT>" ] ) , <EOL> ndelta = float ( PARAMS [ "<STR_LIT>" ] ) , <EOL> npermutations = PARAMS [ "<STR_LIT>" ] , <EOL> method = PARAMS [ "<STR_LIT>" ] , <EOL> use_excel_sam = False ) <EOL> if summary == None : <EOL> E . warn ( "<STR_LIT>" % infile ) <EOL> if cutoffs : <EOL> logs = open ( outfile + "<STR_LIT>" , "<STR_LIT:w>" ) <EOL> logs . write ( "<STR_LIT:\t>" . join ( cutoffs [ <NUM_LIT:0> ] . _fields ) + "<STR_LIT:\n>" ) <EOL> for x in cutoffs : <EOL> logs . write ( "<STR_LIT:\t>" . join ( map ( str , x ) ) + "<STR_LIT:\n>" ) <EOL> logs . close ( ) <EOL> for s in genes : <EOL> outs . write ( "<STR_LIT>" % ( s . probeset , <EOL> s . mean1 , <EOL> s . stddev1 , <EOL> s . mean2 , <EOL> s . stddev2 , <EOL> s . pvalue , <EOL> s . qvalue , <EOL> s . difference , <EOL> s . fold , <EOL> s . called , <EOL> ) ) <EOL> outs . close ( ) <EOL> @ transform ( ( buildExpressionDifferencesSAM , buildExpressionDifferencesTTest ) , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def importExpressionDifferences ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> track , method , control = PExpression . getExpressionMatch ( infile ) <EOL> if track == control : <EOL> outs = open ( outfile , "<STR_LIT:w>" ) <EOL> outs . close ( ) <EOL> return <EOL> tablename = "<STR_LIT>" % ( track , control , method ) <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ transform ( importExpressionDifferences , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def exportDifferentiallyExpressedGenes ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> dbhandle = sqlite3 . connect ( PARAMS [ "<STR_LIT>" ] ) <EOL> track = infile [ : - len ( "<STR_LIT>" ) ] <EOL> if track . endswith ( "<STR_LIT>" ) : <EOL> field = "<STR_LIT>" <EOL> cutoff = float ( PARAMS [ '<STR_LIT>' ] ) <EOL> elif track . endswith ( "<STR_LIT>" ) : <EOL> field = "<STR_LIT>" <EOL> cutoff = float ( PARAMS [ '<STR_LIT>' ] ) <EOL> outs = open ( outfile , "<STR_LIT:w>" ) <EOL> track , method , control = PExpression . getExpressionMatch ( track + "<STR_LIT>" ) <EOL> if track == control : <EOL> outs . close ( ) <EOL> return <EOL> tablename = "<STR_LIT>" % ( track , control , method ) <EOL> statement = '''<STR_LIT>''' % locals ( ) <EOL> cc = dbhandle . cursor ( ) <EOL> cc . execute ( statement ) <EOL> outs . write ( "<STR_LIT>" ) <EOL> for x in cc : <EOL> outs . write ( "<STR_LIT:\t>" . join ( map ( str , x ) ) + "<STR_LIT:\n>" ) <EOL> outs . close ( ) <EOL> @ follows ( buildGeneRegions ) <EOL> @ files ( [ ( "<STR_LIT>" % x , "<STR_LIT>" % x ) for x in TRACKS_MASTER ] ) <EOL> def annotateProbesetTSS ( infile , outfile ) : <EOL> to_cluster = True <EOL> statement = """<STR_LIT>""" <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> @ follows ( buildGeneRegions ) <EOL> @ files ( [ ( "<STR_LIT>" % x , "<STR_LIT>" % x ) for x in TRACKS_NOT_UNSTIM ] ) <EOL> def buildIntervalCounts ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> track = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> cellline , condition , replicate = splitTrack ( track ) <EOL> samfiles_fg , samfiles_bg = [ ] , [ ] <EOL> one = cellline and condition <EOL> two = condition <EOL> if REPLICATES : <EOL> if one : <EOL> for replicate in REPLICATES : <EOL> samfiles_fg . append ( FILEPATTERN % locals ( ) + BAM_SUFFIX ) <EOL> condition = "<STR_LIT>" <EOL> for replicate in REPLICATES : <EOL> samfiles_bg . append ( FILEPATTERN % locals ( ) + BAM_SUFFIX ) <EOL> elif two : <EOL> for cellline in CELLLINES : <EOL> for replicate in REPLICATES : <EOL> samfiles_fg . append ( FILEPATTERN % locals ( ) + BAM_SUFFIX ) <EOL> condition = "<STR_LIT>" <EOL> for cellline in CELLLINES : <EOL> for replicate in REPLICATES : <EOL> samfiles_bg . append ( FILEPATTERN % locals ( ) + BAM_SUFFIX ) <EOL> else : <EOL> samfiles_fg . append ( FILEPATTERN % locals ( ) + BAM_SUFFIX ) <EOL> condition = "<STR_LIT>" <EOL> if os . path . exists ( FILEPATTERN % locals ( ) + BAM_SUFFIX ) : <EOL> samfiles_bg . append ( FILEPATTERN % locals ( ) + BAM_SUFFIX ) <EOL> else : <EOL> samfiles_bg = samfiles_fg <EOL> samfiles_fg = "<STR_LIT:U+002C>" . join ( samfiles_fg ) <EOL> samfiles_bg = "<STR_LIT:U+002C>" . join ( samfiles_bg ) <EOL> tmpfile1 = P . getTempFilename ( os . getcwd ( ) ) + "<STR_LIT>" <EOL> tmpfile2 = P . getTempFilename ( os . getcwd ( ) ) + "<STR_LIT>" <EOL> to_cluster = True <EOL> statement = """<STR_LIT>""" <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> statement = """<STR_LIT>""" <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> os . unlink ( tmpfile1 ) <EOL> os . unlink ( tmpfile2 ) <EOL> @ files_re ( buildIntervalCounts , "<STR_LIT>" , r"<STR_LIT>" ) <EOL> def importIntervalCounts ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> tablename = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> @ files_re ( buildBigwig , combine ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def viewBigwig ( infiles , outfile ) : <EOL> outs = open ( outfile , "<STR_LIT:w>" ) <EOL> outs . write ( "<STR_LIT>" ) <EOL> try : <EOL> os . makedirs ( PARAMS [ "<STR_LIT>" ] ) <EOL> except OSError : <EOL> pass <EOL> for src in infiles : <EOL> dest = os . path . join ( PARAMS [ "<STR_LIT>" ] , src ) <EOL> if not os . path . exists ( dest ) or os . path . getmtime ( src ) > os . path . getmtime ( dest ) : <EOL> shutil . copyfile ( src , dest ) <EOL> track = src [ : - len ( "<STR_LIT>" ) ] <EOL> url = PARAMS [ "<STR_LIT>" ] % src <EOL> outs . write ( '''<STR_LIT>''' % locals ( ) ) <EOL> outs . close ( ) <EOL> @ merge ( importExpressionDifferences , "<STR_LIT>" % PARAMS [ "<STR_LIT:version>" ] ) <EOL> def viewExpressionDifferences ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> dbhandle = sqlite3 . connect ( PARAMS [ "<STR_LIT>" ] ) <EOL> outs = open ( outfile , "<STR_LIT:w>" ) <EOL> for infile in infiles : <EOL> track , method , control = PExpression . getExpressionMatch ( infile [ : - len ( "<STR_LIT>" ) ] ) <EOL> if track == control : continue <EOL> table = "<STR_LIT>" % locals ( ) <EOL> statement = """<STR_LIT>""" % locals ( ) <EOL> cc = dbhandle . cursor ( ) <EOL> cc . execute ( statement ) <EOL> outs . write ( '''<STR_LIT>''' % locals ( ) ) <EOL> for contig , start , end , probeset in cc : <EOL> if contig == "<STR_LIT>" : contig = "<STR_LIT>" <EOL> bed = Bed . Bed ( ) <EOL> bed . contig , bed . start , bed . end = contig , start , end <EOL> bed . fields = [ probeset ] <EOL> outs . write ( "<STR_LIT>" % str ( bed ) ) <EOL> outs . close ( ) <EOL> dest = os . path . join ( PARAMS [ "<STR_LIT>" ] , outfile ) <EOL> shutil . copyfile ( outfile , dest ) <EOL> print "<STR_LIT>" <EOL> print '''<STR_LIT>''' % locals ( ) <EOL> @ follows ( mkdir ( '<STR_LIT>' ) ) <EOL> @ merge ( "<STR_LIT>" , ( "<STR_LIT>" % PARAMS [ "<STR_LIT:version>" ] , "<STR_LIT>" ) ) <EOL> def viewIntervals ( infiles , outfiles ) : <EOL> outfile_bed , outfile_code = outfiles <EOL> outs = open ( outfile_bed , "<STR_LIT:w>" ) <EOL> version = PARAMS [ "<STR_LIT:version>" ] <EOL> for infile in infiles : <EOL> track = infile [ : - len ( "<STR_LIT>" ) ] <EOL> outs . write ( '''<STR_LIT>''' % locals ( ) ) <EOL> with open ( infile , "<STR_LIT:r>" ) as f : <EOL> for bed in Bed . iterator ( f ) : <EOL> if bed . start <= <NUM_LIT:0> : <EOL> bed . start = <NUM_LIT:0> <EOL> outs . write ( str ( bed ) + "<STR_LIT:\n>" ) <EOL> outs . close ( ) <EOL> basename , filename = os . path . split ( outfile_bed ) <EOL> dest = os . path . join ( PARAMS [ "<STR_LIT>" ] , filename ) <EOL> try : <EOL> os . makedirs ( PARAMS [ "<STR_LIT>" ] ) <EOL> except OSError : <EOL> pass <EOL> shutil . copyfile ( outfile_bed , dest ) <EOL> filename = re . sub ( "<STR_LIT>" , "<STR_LIT>" , dest ) <EOL> outs = open ( outfile_code , "<STR_LIT:w>" ) <EOL> outs . write ( "<STR_LIT>" ) <EOL> outs . write ( "<STR_LIT>" % locals ( ) ) <EOL> outs . close ( ) <EOL> @ files ( [ ( None , PARAMS [ "<STR_LIT>" ] ) , ] ) <EOL> def createGO ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> @ files_re ( createGO , "<STR_LIT>" , r"<STR_LIT>" ) <EOL> def createGOSlim ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> @ follows ( buildGeneRegions ) <EOL> @ files ( [ ( "<STR_LIT>" % x , "<STR_LIT>" % ( x , y ) ) for x , y in itertools . product ( TRACKS_ANNOTATOR , ( TARGET_TSS , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ) ) ] ) <EOL> def makeAnnotatorDistance ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> track , workspace = outfile [ : - len ( "<STR_LIT>" ) ] . split ( "<STR_LIT:->" ) <EOL> if workspace == "<STR_LIT>" : <EOL> builder = "<STR_LIT>" <EOL> workspace = TARGET_GENESET <EOL> elif workspace == "<STR_LIT>" : <EOL> builder = "<STR_LIT>" <EOL> workspace = TARGET_GENESET <EOL> elif workspace == "<STR_LIT>" : <EOL> builder = "<STR_LIT>" <EOL> workspace = TARGET_GENESET <EOL> else : <EOL> builder = "<STR_LIT>" <EOL> PAnnotator . makeAnnotatorDistance ( infile , outfile , builder , workspace , <EOL> workspace_label = "<STR_LIT>" ) <EOL> @ follows ( buildGeneRegions ) <EOL> @ files ( [ ( "<STR_LIT>" % x , <EOL> "<STR_LIT>" % ( x ) ) for x in TRACKS_MASTER ] ) <EOL> def makeAnnotatorDistanceExpression ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> track , workspace = outfile [ : - len ( "<STR_LIT>" ) ] . split ( "<STR_LIT:->" ) <EOL> builder = "<STR_LIT>" <EOL> workspace = TARGET_GENESET <EOL> tmpannotations = PAnnotator . buildAnnotatorDistanceAnnotations ( "<STR_LIT>" ) <EOL> PAnnotator . makeAnnotatorDistance ( infile , <EOL> outfile , <EOL> builder , <EOL> workspace , <EOL> workspace_label = "<STR_LIT>" , <EOL> annotations = tmpannotations , <EOL> ) <EOL> os . unlink ( tmpannotations ) <EOL> @ follows ( buildGeneRegions ) <EOL> @ files ( [ ( "<STR_LIT>" % x , <EOL> "<STR_LIT>" % ( x ) ) for x in TRACKS_MASTER ] ) <EOL> def makeAnnotatorDistanceWithoutPromotorExpression ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> track , workspace = outfile [ : - len ( "<STR_LIT>" ) ] . split ( "<STR_LIT:->" ) <EOL> builder = "<STR_LIT>" <EOL> tmpfilename = P . getTempFilename ( "<STR_LIT:.>" ) <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> workspace = tmpfilename <EOL> tmpannotations = PAnnotator . buildAnnotatorDistanceAnnotations ( "<STR_LIT>" ) <EOL> PAnnotator . makeAnnotatorDistance ( infile , <EOL> outfile , <EOL> builder , <EOL> workspace , <EOL> workspace_label = "<STR_LIT>" , <EOL> annotations = tmpannotations , <EOL> ) <EOL> os . unlink ( tmpannotations ) <EOL> @ follows ( buildGeneRegions , buildAnnotatorGC ) <EOL> @ files ( [ ( "<STR_LIT>" % x , "<STR_LIT>" % x ) for x in TRACKS_ANNOTATOR ] ) <EOL> def makeAnnotatorArchitecture ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PAnnotator . makeAnnotatorArchitecture ( infile , outfile ) <EOL> @ follows ( buildGeneRegions , buildAnnotatorGC ) <EOL> @ files ( [ ( ( "<STR_LIT>" % x , "<STR_LIT>" ) , "<STR_LIT>" % x ) for x in TRACKS_ANNOTATOR ] ) <EOL> def makeAnnotatorGWAS ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PAnnotator . makeAnnotatorRegionsOfInterest ( infiles , outfile ) <EOL> @ follows ( buildGeneRegions , buildAnnotatorGC ) <EOL> @ files ( [ ( ( "<STR_LIT>" % x , "<STR_LIT>" ) , "<STR_LIT>" % x ) for x in TRACKS_ANNOTATOR ] ) <EOL> def makeAnnotatorGWASWithMotif ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PAnnotator . makeAnnotatorRegionsOfInterest ( infiles , <EOL> outfile , <EOL> with_motif = PARAMS [ '<STR_LIT>' ] ) <EOL> @ follows ( buildGeneRegions , buildAnnotatorGC ) <EOL> @ files ( [ ( ( "<STR_LIT>" % x , "<STR_LIT>" ) , "<STR_LIT>" % x ) for x in TRACKS_ANNOTATOR ] ) <EOL> def makeAnnotatorGWASWithoutMotif ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PAnnotator . makeAnnotatorRegionsOfInterest ( infiles , <EOL> outfile , <EOL> without_motif = PARAMS [ '<STR_LIT>' ] ) <EOL> @ follows ( buildGeneRegions , buildAnnotatorGC ) <EOL> @ files ( [ ( ( "<STR_LIT>" % x , "<STR_LIT>" ) , "<STR_LIT>" % x ) for x in TRACKS_ANNOTATOR ] ) <EOL> def makeAnnotatorSelection ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PAnnotator . makeAnnotatorRegionsOfInterest ( infiles , outfile ) <EOL> @ follows ( exportUCSCEncodeTracks , buildAnnotatorGC ) <EOL> @ files ( [ ( ( "<STR_LIT>" % x , "<STR_LIT>" ) , "<STR_LIT>" % x ) for x in TRACKS_ANNOTATOR ] ) <EOL> def makeAnnotatorTracks ( infiles , outfile ) : <EOL> PAnnotator . makeAnnotatorTracks ( infiles , outfile ) <EOL> @ follows ( exportRegionsOfInterest , buildAnnotatorGC ) <EOL> @ files ( [ ( ( "<STR_LIT>" % x , "<STR_LIT>" ) , "<STR_LIT>" % x ) for x in TRACKS_ANNOTATOR ] ) <EOL> def makeAnnotatorRegionsOfInterest ( infiles , outfile ) : <EOL> PAnnotator . makeAnnotatorRegionsOfInterest ( infiles , outfile ) <EOL> @ follows ( exportUCSCEncodeTracks , buildAnnotatorGC ) <EOL> @ files ( [ ( ( "<STR_LIT>" % x , "<STR_LIT>" ) , "<STR_LIT>" % x ) for x in TRACKS_ANNOTATOR ] ) <EOL> def makeAnnotatorPromotors ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> infile , infile_annotations = infiles <EOL> tmpdir = P . getTempDir ( dir = os . getcwd ( ) ) <EOL> annotations = PAnnotator . buildAnnotatorAnnotations ( tmpdir , outfile , bedfiles = ( infile_annotations , ) ) <EOL> workspaces , synonyms = PAnnotator . buildAnnotatorWorkSpace ( tmpdir , outfile , <EOL> workspaces = ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> gc_control = True ) <EOL> segments = PAnnotator . buildAnnotatorSegments ( tmpdir , infile , outfile ) <EOL> PAnnotator . runAnnotator ( tmpdir , outfile , annotations , segments , workspaces , synonyms ) <EOL> shutil . rmtree ( tmpdir ) <EOL> @ merge ( makeAnnotatorTracks , <EOL> "<STR_LIT>" ) <EOL> def importAnnotatorTracks ( infiles , outfile ) : <EOL> PAnnotator . genericImportAnnotator ( infiles , outfile , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:all>" , "<STR_LIT:all>" , "<STR_LIT>" ) <EOL> @ merge ( makeAnnotatorArchitecture , <EOL> "<STR_LIT>" ) <EOL> def importAnnotatorArchitecture ( infiles , outfile ) : <EOL> PAnnotator . genericImportAnnotator ( infiles , outfile , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:all>" , "<STR_LIT:all>" , "<STR_LIT>" ) <EOL> @ merge ( makeAnnotatorRegionsOfInterest , <EOL> "<STR_LIT>" ) <EOL> def importAnnotatorRegionsOfInterest ( infiles , outfile ) : <EOL> PAnnotator . genericImportAnnotator ( infiles , outfile , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:all>" , "<STR_LIT:all>" , "<STR_LIT>" ) <EOL> @ merge ( makeAnnotatorGWAS , <EOL> "<STR_LIT>" ) <EOL> def importAnnotatorGWAS ( infiles , outfile ) : <EOL> PAnnotator . genericImportAnnotator ( infiles , outfile , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:all>" , "<STR_LIT:all>" , "<STR_LIT>" ) <EOL> @ merge ( makeAnnotatorSelection , <EOL> "<STR_LIT>" ) <EOL> def importAnnotatorSelection ( infiles , outfile ) : <EOL> PAnnotator . genericImportAnnotator ( infiles , outfile , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:all>" , "<STR_LIT:all>" , "<STR_LIT>" ) <EOL> @ follows ( importAnnotatorRegionsOfInterest , <EOL> importAnnotatorGWAS , <EOL> importAnnotatorSelection ) <EOL> def annotator_regions ( ) : pass <EOL> @ merge ( makeAnnotatorPromotors , <EOL> "<STR_LIT>" ) <EOL> def importAnnotatorPromotors ( infiles , outfile ) : <EOL> PAnnotator . genericImportAnnotator ( infiles , outfile , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:all>" , "<STR_LIT:all>" , "<STR_LIT>" ) <EOL> @ follows ( buildGeneRegions ) <EOL> @ files ( [ ( "<STR_LIT>" % x , "<STR_LIT>" % x , <EOL> PARAMS [ "<STR_LIT>" ] , "<STR_LIT>" ) for x in TRACKS_ANNOTATOR ] ) <EOL> def makeAnnotatorPromotorsGO ( infile , outfile , gofile , workspace ) : <EOL> PAnnotator . makeAnnotatorGO ( infile , outfile , gofile , workspace ) <EOL> @ follows ( buildGeneRegions ) <EOL> @ files ( [ ( "<STR_LIT>" % x , "<STR_LIT>" % x , <EOL> PARAMS [ "<STR_LIT>" ] , "<STR_LIT>" ) for x in TRACKS_ANNOTATOR ] ) <EOL> def makeAnnotatorPromotorsGOSlim ( infile , outfile , gofile , workspace ) : <EOL> PAnnotator . makeAnnotatorGO ( infile , outfile , gofile , workspace ) <EOL> @ follows ( buildGeneRegions ) <EOL> @ files ( [ ( "<STR_LIT>" % x , "<STR_LIT>" % x , <EOL> PARAMS [ "<STR_LIT>" ] , "<STR_LIT>" ) for x in TRACKS_ANNOTATOR ] ) <EOL> def makeAnnotatorTerritoriesGO ( infile , outfile , gofile , workspace ) : <EOL> PAnnotator . makeAnnotatorGO ( infile , outfile , gofile , workspace ) <EOL> @ follows ( buildGeneRegions ) <EOL> @ files ( [ ( "<STR_LIT>" % x , "<STR_LIT>" % x , <EOL> PARAMS [ "<STR_LIT>" ] , "<STR_LIT>" ) for x in TRACKS_ANNOTATOR ] ) <EOL> def makeAnnotatorTerritoriesGOSlim ( infile , outfile , gofile , workspace ) : <EOL> PAnnotator . makeAnnotatorGO ( infile , outfile , gofile , workspace ) <EOL> @ merge ( makeAnnotatorPromotorsGO , "<STR_LIT>" ) <EOL> def importAnnotatorPromotorsGO ( infiles , outfile ) : <EOL> PAnnotator . genericImportAnnotator ( infiles , outfile , <EOL> "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT:all>" , "<STR_LIT:all>" , <EOL> fdr_method = "<STR_LIT>" ) <EOL> @ merge ( makeAnnotatorPromotorsGOSlim , "<STR_LIT>" ) <EOL> def importAnnotatorPromotorsGOSlim ( infiles , outfile ) : <EOL> PAnnotator . genericImportAnnotator ( infiles , outfile , <EOL> "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT:all>" , "<STR_LIT:all>" , <EOL> fdr_method = "<STR_LIT>" ) <EOL> @ merge ( makeAnnotatorTerritoriesGO , "<STR_LIT>" ) <EOL> def importAnnotatorTerritoriesGO ( infiles , outfile ) : <EOL> PAnnotator . genericImportAnnotator ( infiles , outfile , <EOL> "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT:all>" , "<STR_LIT:all>" , <EOL> fdr_method = "<STR_LIT>" ) <EOL> @ merge ( makeAnnotatorTerritoriesGOSlim , "<STR_LIT>" ) <EOL> def importAnnotatorTerritoriesGOSlim ( infiles , outfile ) : <EOL> PAnnotator . genericImportAnnotator ( infiles , outfile , <EOL> "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT:all>" , "<STR_LIT:all>" , <EOL> fdr_method = "<STR_LIT>" ) <EOL> if PARAMS [ "<STR_LIT>" ] : <EOL> @ follows ( buildGeneRegions ) <EOL> @ files ( [ ( "<STR_LIT>" , <EOL> "<STR_LIT>" % x , <EOL> x , <EOL> PARAMS [ "<STR_LIT>" ] , <EOL> "<STR_LIT>" ) for x in TRACKS_ROI ] ) <EOL> def makeAnnotatorROIGOSlim ( infile , outfile , roi_class , gofile , workspace ) : <EOL> '''<STR_LIT>''' <EOL> PAnnotator . makeAnnotatorROIGO ( roi_class , outfile , gofile , workspace , overlap = None ) <EOL> @ follows ( buildGeneRegions ) <EOL> @ files ( [ ( "<STR_LIT>" , <EOL> "<STR_LIT>" % x , <EOL> x , <EOL> PARAMS [ "<STR_LIT>" ] , <EOL> "<STR_LIT>" ) for x in TRACKS_ROI ] ) <EOL> def makeAnnotatorROIGO ( infile , outfile , roi_class , gofile , workspace ) : <EOL> '''<STR_LIT>''' <EOL> PAnnotator . makeAnnotatorROIGO ( roi_class , outfile , gofile , workspace , overlap = None ) <EOL> @ follows ( buildGeneRegions ) <EOL> @ files ( [ ( "<STR_LIT>" , <EOL> "<STR_LIT>" % x , <EOL> x , <EOL> PARAMS [ "<STR_LIT>" ] , <EOL> "<STR_LIT>" ) for x in TRACKS_ROI ] ) <EOL> def makeAnnotatorROIOverlapGOSlim ( infile , outfile , roi_class , gofile , workspace ) : <EOL> '''<STR_LIT>''' <EOL> PAnnotator . makeAnnotatorROIGO ( roi_class , outfile , gofile , workspace , overlap = PARAMS [ "<STR_LIT>" ] ) <EOL> @ follows ( buildGeneRegions ) <EOL> @ files ( [ ( "<STR_LIT>" , <EOL> "<STR_LIT>" % x , <EOL> x , <EOL> PARAMS [ "<STR_LIT>" ] , <EOL> "<STR_LIT>" ) for x in TRACKS_ROI ] ) <EOL> def makeAnnotatorROIOverlapGO ( infile , outfile , roi_class , gofile , workspace ) : <EOL> '''<STR_LIT>''' <EOL> PAnnotator . makeAnnotatorROIGO ( roi_class , outfile , gofile , workspace , overlap = PARAMS [ "<STR_LIT>" ] ) <EOL> @ merge ( makeAnnotatorROIGO , "<STR_LIT>" ) <EOL> def importAnnotatorROIGO ( infiles , outfile ) : <EOL> PAnnotator . genericImportAnnotator ( infiles , outfile , <EOL> "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT:all>" , "<STR_LIT:all>" , <EOL> fdr_method = "<STR_LIT>" ) <EOL> @ merge ( makeAnnotatorROIGOSlim , "<STR_LIT>" ) <EOL> def importAnnotatorROIGOSlim ( infiles , outfile ) : <EOL> PAnnotator . genericImportAnnotator ( infiles , outfile , <EOL> "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT:all>" , "<STR_LIT:all>" , <EOL> fdr_method = "<STR_LIT>" ) <EOL> @ merge ( makeAnnotatorROIOverlapGO , "<STR_LIT>" ) <EOL> def importAnnotatorROIOverlapGO ( infiles , outfile ) : <EOL> PAnnotator . genericImportAnnotator ( infiles , outfile , <EOL> "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT:all>" , "<STR_LIT:all>" , <EOL> fdr_method = "<STR_LIT>" ) <EOL> @ merge ( makeAnnotatorROIOverlapGOSlim , "<STR_LIT>" ) <EOL> def importAnnotatorROIOverlapGOSlim ( infiles , outfile ) : <EOL> PAnnotator . genericImportAnnotator ( infiles , outfile , <EOL> "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT:all>" , "<STR_LIT:all>" , <EOL> fdr_method = "<STR_LIT>" ) <EOL> @ follows ( makeAnnotatorROIGO , importAnnotatorROIGO , <EOL> makeAnnotatorROIGOSlim , importAnnotatorROIGOSlim , <EOL> makeAnnotatorROIOverlapGO , importAnnotatorROIOverlapGO , <EOL> makeAnnotatorROIOverlapGOSlim , importAnnotatorROIOverlapGOSlim , ) <EOL> def annotator_roi ( ) : <EOL> pass <EOL> else : <EOL> @ follows ( buildGeneSet ) <EOL> def annotator_roi ( ) : <EOL> pass <EOL> if PARAMS [ "<STR_LIT>" ] : <EOL> @ follows ( buildGeneRegions , buildAnnotatorGC ) <EOL> @ files ( [ ( "<STR_LIT>" % x , "<STR_LIT>" % x ) for x in TRACKS_ANNOTATOR ] ) <EOL> def makeAnnotatorArchitectureWithMotif ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PAnnotator . makeAnnotatorArchitecture ( infile , outfile , with_motif = PARAMS [ '<STR_LIT>' ] ) <EOL> @ follows ( buildGeneRegions , buildAnnotatorGC ) <EOL> @ files ( [ ( "<STR_LIT>" % x , "<STR_LIT>" % x ) for x in TRACKS_ANNOTATOR ] ) <EOL> def makeAnnotatorArchitectureWithoutMotif ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PAnnotator . makeAnnotatorArchitecture ( infile , outfile , without_motif = PARAMS [ '<STR_LIT>' ] ) <EOL> @ follows ( exportUCSCEncodeTracks , buildAnnotatorGC ) <EOL> @ files ( [ ( ( "<STR_LIT>" % x , "<STR_LIT>" ) , "<STR_LIT>" % x ) for x in TRACKS_ANNOTATOR ] ) <EOL> def makeAnnotatorTracksWithMotif ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PAnnotator . makeAnnotatorTracks ( infiles , outfile , with_motif = PARAMS [ '<STR_LIT>' ] ) <EOL> @ follows ( exportUCSCEncodeTracks , buildAnnotatorGC ) <EOL> @ files ( [ ( ( "<STR_LIT>" % x , "<STR_LIT>" ) , "<STR_LIT>" % x ) for x in TRACKS_ANNOTATOR ] ) <EOL> def makeAnnotatorTracksWithoutMotif ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PAnnotator . makeAnnotatorTracks ( infiles , outfile , without_motif = PARAMS [ '<STR_LIT>' ] ) <EOL> @ follows ( exportRegionsOfInterest , buildAnnotatorGC ) <EOL> @ files ( [ ( ( "<STR_LIT>" % x , "<STR_LIT>" ) , "<STR_LIT>" % x ) for x in TRACKS_ANNOTATOR ] ) <EOL> def makeAnnotatorRegionsOfInterestWithMotif ( infiles , outfile ) : <EOL> PAnnotator . makeAnnotatorRegionsOfInterest ( infiles , outfile , with_motif = PARAMS [ '<STR_LIT>' ] ) <EOL> @ follows ( exportRegionsOfInterest , buildAnnotatorGC ) <EOL> @ files ( [ ( ( "<STR_LIT>" % x , "<STR_LIT>" ) , "<STR_LIT>" % x ) for x in TRACKS_ANNOTATOR ] ) <EOL> def makeAnnotatorRegionsOfInterestWithoutMotif ( infiles , outfile ) : <EOL> PAnnotator . makeAnnotatorRegionsOfInterest ( infiles , outfile , without_motif = PARAMS [ '<STR_LIT>' ] ) <EOL> @ merge ( makeAnnotatorArchitectureWithMotif , <EOL> "<STR_LIT>" ) <EOL> def importAnnotatorArchitectureWithMotif ( infiles , outfile ) : <EOL> PAnnotator . genericImportAnnotator ( infiles , outfile , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:all>" , "<STR_LIT:all>" , "<STR_LIT>" ) <EOL> @ merge ( makeAnnotatorArchitectureWithoutMotif , <EOL> "<STR_LIT>" ) <EOL> def importAnnotatorArchitectureWithoutMotif ( infiles , outfile ) : <EOL> PAnnotator . genericImportAnnotator ( infiles , outfile , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:all>" , "<STR_LIT:all>" , "<STR_LIT>" ) <EOL> @ merge ( makeAnnotatorTracksWithMotif , <EOL> "<STR_LIT>" ) <EOL> def importAnnotatorTracksWithMotif ( infiles , outfile ) : <EOL> PAnnotator . genericImportAnnotator ( infiles , outfile , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:all>" , "<STR_LIT:all>" , "<STR_LIT>" ) <EOL> @ merge ( makeAnnotatorTracksWithoutMotif , <EOL> "<STR_LIT>" ) <EOL> def importAnnotatorTracksWithoutMotif ( infiles , outfile ) : <EOL> PAnnotator . genericImportAnnotator ( infiles , outfile , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:all>" , "<STR_LIT:all>" , "<STR_LIT>" ) <EOL> @ merge ( makeAnnotatorRegionsOfInterestWithMotif , <EOL> "<STR_LIT>" ) <EOL> def importAnnotatorRegionsOfInterestWithMotif ( infiles , outfile ) : <EOL> PAnnotator . genericImportAnnotator ( infiles , outfile , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:all>" , "<STR_LIT:all>" , "<STR_LIT>" ) <EOL> @ merge ( makeAnnotatorRegionsOfInterestWithoutMotif , <EOL> "<STR_LIT>" ) <EOL> def importAnnotatorRegionsOfInterestWithoutMotif ( infiles , outfile ) : <EOL> PAnnotator . genericImportAnnotator ( infiles , outfile , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:all>" , "<STR_LIT:all>" , "<STR_LIT>" ) <EOL> @ follows ( makeAnnotatorArchitectureWithMotif , importAnnotatorArchitectureWithMotif , <EOL> makeAnnotatorArchitectureWithoutMotif , importAnnotatorArchitectureWithoutMotif , <EOL> makeAnnotatorTracksWithMotif , importAnnotatorTracksWithMotif , <EOL> makeAnnotatorTracksWithoutMotif , importAnnotatorTracksWithoutMotif , <EOL> makeAnnotatorRegionsOfInterestWithMotif , importAnnotatorRegionsOfInterestWithMotif , <EOL> makeAnnotatorRegionsOfInterestWithoutMotif , importAnnotatorRegionsOfInterestWithoutMotif ) <EOL> def annotator_motifs ( ) : <EOL> pass <EOL> else : <EOL> @ follows ( buildGeneSet ) <EOL> def annotator_motifs ( ) : <EOL> pass <EOL> if PARAMS [ "<STR_LIT>" ] : <EOL> @ follows ( exportRegionsOfInterest , buildAnnotatorGC ) <EOL> @ files ( [ ( ( "<STR_LIT>" % x , "<STR_LIT>" ) , "<STR_LIT>" % x ) for x in TRACKS_ANNOTATOR ] ) <EOL> def makeAnnotatorRegionsOfInterestTop ( infiles , outfile ) : <EOL> PAnnotator . makeAnnotatorRegionsOfInterest ( infiles , outfile , proportion = PARAMS [ '<STR_LIT>' ] ) <EOL> @ follows ( exportRegionsOfInterest , buildAnnotatorGC ) <EOL> @ files ( [ ( ( "<STR_LIT>" % x , "<STR_LIT>" ) , "<STR_LIT>" % x ) for x in TRACKS_ANNOTATOR ] ) <EOL> def makeAnnotatorRegionsOfInterestBottom ( infiles , outfile ) : <EOL> PAnnotator . makeAnnotatorRegionsOfInterest ( infiles , outfile , proportion = - PARAMS [ '<STR_LIT>' ] ) <EOL> @ merge ( makeAnnotatorRegionsOfInterestTop , <EOL> "<STR_LIT>" ) <EOL> def importAnnotatorRegionsOfInterestTop ( infiles , outfile ) : <EOL> PAnnotator . genericImportAnnotator ( infiles , outfile , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:all>" , "<STR_LIT:all>" , "<STR_LIT>" ) <EOL> @ merge ( makeAnnotatorRegionsOfInterestBottom , <EOL> "<STR_LIT>" ) <EOL> def importAnnotatorRegionsOfInterestBottom ( infiles , outfile ) : <EOL> PAnnotator . genericImportAnnotator ( infiles , outfile , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:all>" , "<STR_LIT:all>" , "<STR_LIT>" ) <EOL> @ follows ( makeAnnotatorRegionsOfInterestTop , <EOL> importAnnotatorRegionsOfInterestTop , <EOL> makeAnnotatorRegionsOfInterestBottom , <EOL> importAnnotatorRegionsOfInterestBottom ) <EOL> def annotator_proportion ( ) : <EOL> pass <EOL> else : <EOL> @ follows ( buildGeneSet ) <EOL> def annotator_proportion ( ) : <EOL> pass <EOL> @ follows ( buildGeneRegions ) <EOL> @ files ( [ ( ( "<STR_LIT>" % x , TARGET_TRANSCRIPTS_TSS ) , "<STR_LIT>" % x ) for x in TRACKS_ANNOTATE ] ) <EOL> def annotateTSSIntervalDistance ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> infile_bed , infile_tss = infiles <EOL> to_cluster = True <EOL> statement = """<STR_LIT>""" <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> @ files_re ( annotateTSSIntervalDistance , <EOL> "<STR_LIT>" , <EOL> r"<STR_LIT>" ) <EOL> def importTSSIntervalDistance ( infile , outfile ) : <EOL> tablename = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> @ follows ( buildGeneRegions ) <EOL> @ files ( [ ( ( "<STR_LIT>" % x , TARGET_TRANSCRIPTS_TSS ) , "<STR_LIT>" % x ) for x in TRACKS_ANNOTATE ] ) <EOL> def annotateTSSIntervalAssociations ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> infile_bed , infile_tss = infiles <EOL> to_cluster = True <EOL> statement = """<STR_LIT>""" <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> @ files_re ( annotateTSSIntervalAssociations , <EOL> "<STR_LIT>" , <EOL> r"<STR_LIT>" ) <EOL> def importTSSIntervalAssociations ( infile , outfile ) : <EOL> tablename = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> @ files ( PARAMS [ "<STR_LIT>" ] , "<STR_LIT>" ) <EOL> def buildDiseaseIntervals ( infile , outfile ) : <EOL> reader = csv . reader ( open ( infile , "<STR_LIT>" ) ) <EOL> fasta = IndexedFasta . IndexedFasta ( FILENAME_GENOME ) <EOL> outs = open ( outfile , "<STR_LIT:w>" ) <EOL> for row in reader : <EOL> if row [ <NUM_LIT:0> ] == "<STR_LIT>" : <EOL> headers = row <EOL> continue <EOL> loc , chrom , start , end = row [ : <NUM_LIT:4> ] <EOL> if loc == "<STR_LIT>" : continue <EOL> start , end = int ( start ) - <NUM_LIT:1> , int ( end ) <EOL> contig = fasta . getToken ( re . sub ( "<STR_LIT>" , "<STR_LIT>" , chrom ) ) <EOL> outs . write ( "<STR_LIT>" % ( contig , start , end , loc ) ) <EOL> outs . close ( ) <EOL> @ files ( ( ( PARAMS [ "<STR_LIT>" ] , "<STR_LIT>" ) , ) ) <EOL> def importExpressionWang ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> headers = ( <EOL> ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ( "<STR_LIT>" , "<STR_LIT>" , ) , <EOL> ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ( "<STR_LIT>" , "<STR_LIT>" , ) , <EOL> ( "<STR_LIT>" , "<STR_LIT>" ) , ) <EOL> old_headers = set ( [ x [ <NUM_LIT:0> ] for x in headers ] ) <EOL> new_headers = [ x [ <NUM_LIT:1> ] for x in headers ] <EOL> take = [ ] <EOL> outs = P . getTempFile ( ) <EOL> first = True <EOL> reader = csv . reader ( open ( infile , "<STR_LIT>" ) ) <EOL> for row in reader : <EOL> if first : <EOL> first = False <EOL> for x , old_header in enumerate ( row ) : <EOL> if old_header in old_headers : take . append ( x ) <EOL> outs . write ( "<STR_LIT:\t>" . join ( new_headers ) + "<STR_LIT:\n>" ) <EOL> else : <EOL> new_row = [ ] <EOL> for x in take : <EOL> if row [ x ] . strip ( ) != "<STR_LIT>" : <EOL> new_row . append ( row [ x ] . strip ( ) ) <EOL> else : <EOL> new_row . append ( "<STR_LIT>" ) <EOL> outs . write ( "<STR_LIT:\t>" . join ( new_row ) + "<STR_LIT:\n>" ) <EOL> outs . close ( ) <EOL> tmpname = outs . name <EOL> tablename = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> os . unlink ( tmpname ) <EOL> @ files ( ( ( PARAMS [ "<STR_LIT>" ] , "<STR_LIT>" ) , ) ) <EOL> def importExpressionKovalenko ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> dbhandle = sqlite3 . connect ( PARAMS [ "<STR_LIT>" ] ) <EOL> cc = dbhandle . cursor ( ) <EOL> map_name2id = dict ( cc . execute ( "<STR_LIT>" ) . fetchall ( ) ) <EOL> headers = ( <EOL> ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ) <EOL> old_headers = set ( [ x [ <NUM_LIT:0> ] for x in headers ] ) <EOL> new_headers = [ x [ <NUM_LIT:1> ] for x in headers ] <EOL> take = [ ] <EOL> outs = P . getTempFile ( ) <EOL> first = True <EOL> counts = E . Counter ( ) <EOL> reader = csv . reader ( open ( infile , "<STR_LIT>" ) ) <EOL> notfound = set ( ) <EOL> for row in reader : <EOL> if first : <EOL> first = False <EOL> for x , old_header in enumerate ( row ) : <EOL> if old_header in old_headers : take . append ( x ) <EOL> outs . write ( "<STR_LIT:\t>" . join ( new_headers ) + "<STR_LIT:\n>" ) <EOL> else : <EOL> skip = False <EOL> new_row = [ ] <EOL> for x in take : <EOL> if headers [ x ] [ <NUM_LIT:0> ] . startswith ( "<STR_LIT>" ) : <EOL> foldval = float ( row [ x ] ) <EOL> if foldval < <NUM_LIT:0> : <EOL> row [ x ] = str ( - <NUM_LIT:1.0> / foldval ) <EOL> elif headers [ x ] [ <NUM_LIT:0> ] . startswith ( "<STR_LIT>" ) : <EOL> qvalue = float ( row [ x ] ) <EOL> row [ x ] = str ( qvalue / <NUM_LIT> ) <EOL> elif headers [ x ] [ <NUM_LIT:0> ] . startswith ( "<STR_LIT>" ) : <EOL> parts = [ c . upper ( ) . strip ( ) for c in row [ x ] . split ( "<STR_LIT>" ) ] <EOL> for part in parts : <EOL> if part in map_name2id : <EOL> row [ x ] = map_name2id [ part ] . upper ( ) <EOL> counts . found += <NUM_LIT:1> <EOL> break <EOL> else : <EOL> counts . notfound += <NUM_LIT:1> <EOL> notfound . add ( row [ x ] ) <EOL> skip = True <EOL> break <EOL> if row [ x ] . strip ( ) != "<STR_LIT>" : <EOL> new_row . append ( row [ x ] . strip ( ) ) <EOL> else : <EOL> new_row . append ( "<STR_LIT>" ) <EOL> if skip : continue <EOL> outs . write ( "<STR_LIT:\t>" . join ( new_row ) + "<STR_LIT:\n>" ) <EOL> outs . close ( ) <EOL> tmpname = outs . name <EOL> tablename = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> os . unlink ( tmpname ) <EOL> E . info ( "<STR_LIT:%s>" % str ( counts ) ) <EOL> if notfound : <EOL> E . warn ( "<STR_LIT>" % str ( notfound ) ) <EOL> @ transform ( importExpressionDifferences , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def runExpressionGOSlim ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PExpression . runGO ( infile , outfile , PARAMS [ "<STR_LIT>" ] ) <EOL> @ transform ( importExpressionDifferences , <EOL> suffix ( "<STR_LIT>" ) , <EOL> "<STR_LIT>" ) <EOL> def runExpressionGO ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> PExpression . runGO ( infile , outfile , PARAMS [ "<STR_LIT>" ] ) <EOL> @ transform ( runExpressionGO , <EOL> suffix ( "<STR_LIT>" ) , <EOL> ( "<STR_LIT>" ) ) <EOL> def importExpressionGO ( infile , outfile ) : <EOL> PExpression . importGO ( infile , outfile , "<STR_LIT>" ) <EOL> @ transform ( runExpressionGOSlim , <EOL> suffix ( "<STR_LIT>" ) , <EOL> ( "<STR_LIT>" ) ) <EOL> def importExpressionGOSlim ( infile , outfile ) : <EOL> PExpression . importGO ( infile , outfile , "<STR_LIT>" ) <EOL> @ transform ( buildGeneSet , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def importOverlapRegionsOfInterestEnsembl ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> tmpfilename = P . getTempFilename ( "<STR_LIT:.>" ) <EOL> PGeneset . buildOverlapWithEnsembl ( infile , <EOL> tmpfilename , <EOL> "<STR_LIT>" ) <EOL> tablename = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> os . unlink ( tmpfilename ) <EOL> os . unlink ( tmpfilename + "<STR_LIT>" ) <EOL> @ transform ( buildGeneSet , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def importGWASEnsembl ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> tmpfilename = P . getTempFilename ( "<STR_LIT:.>" ) <EOL> PGeneset . buildOverlapWithEnsembl ( infile , <EOL> tmpfilename , <EOL> "<STR_LIT>" ) <EOL> tablename = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> os . unlink ( tmpfilename ) <EOL> os . unlink ( tmpfilename + "<STR_LIT>" ) <EOL> @ follows ( buildGeneRegions , importRegionsOfInterest ) <EOL> @ files ( [ ( ( "<STR_LIT>" % x , "<STR_LIT>" % y ) , "<STR_LIT>" % ( x , y ) ) <EOL> for x , y in itertools . product ( TRACKS_ANNOTATOR , TRACKS_REGIONS ) ] ) <EOL> def importOverlapRegions ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> infile , bedfile = infiles <EOL> tablename = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> to_cluster = True <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ follows ( importSNPsOfInterest , runMACS ) <EOL> @ transform ( "<STR_LIT>" , <EOL> regex ( r"<STR_LIT>" ) , <EOL> inputs ( ( r"<STR_LIT>" , "<STR_LIT>" , ) ) , <EOL> r"<STR_LIT>" ) <EOL> def buildSNPCoverage ( infiles , outfile ) : <EOL> infile , other = infiles <EOL> dbhandle = sqlite3 . connect ( PARAMS [ "<STR_LIT>" ] ) <EOL> cc = dbhandle . cursor ( ) <EOL> tmpfilename = P . getTempFilename ( ) <EOL> outf = open ( outfile , "<STR_LIT:w>" ) <EOL> fields = pysam . Pileup . PileupSubstitution . _fields <EOL> outf . write ( "<STR_LIT>" % "<STR_LIT:\t>" . join ( fields ) ) <EOL> samfile = pysam . Samfile ( infile , "<STR_LIT:rb>" ) <EOL> for snp , contig , pos , reference in cc . execute ( "<STR_LIT>" ) : <EOL> start = pos - PARAMS [ "<STR_LIT>" ] <EOL> end = pos + PARAMS [ "<STR_LIT>" ] + <NUM_LIT:1> <EOL> nreads = len ( list ( samfile . fetch ( contig , start , end ) ) ) <EOL> start = pos <EOL> end = pos + <NUM_LIT:1> <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ** dict ( locals ( ) . items ( ) + PARAMS . items ( ) ) ) <EOL> for s in pysam . Pileup . iterate ( open ( tmpfilename , "<STR_LIT:r>" ) ) : <EOL> if s . position == pos : <EOL> if s . reference_base != reference : <EOL> E . warn ( "<STR_LIT>" % ( snp , reference , s . reference_base ) ) <EOL> outf . write ( "<STR_LIT>" % ( snp , <EOL> nreads , <EOL> "<STR_LIT:\t>" . join ( map ( str , s ) ) ) ) <EOL> break <EOL> else : <EOL> outf . write ( "<STR_LIT>" % ( snp , <EOL> nreads , <EOL> "<STR_LIT:\t>" . join ( [ "<STR_LIT>" ] * len ( fields ) ) ) ) <EOL> os . unlink ( tmpfilename ) <EOL> @ transform ( buildSNPCoverage , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def importSNPCoverage ( infile , outfile ) : <EOL> tablename = outfile [ : - len ( "<STR_LIT>" ) ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ follows ( indexGenome , buildGenome , <EOL> buildGeneRegions , buildGeneSet , buildPromotorRegions , buildTSSRegions , <EOL> buildRepeats , buildTranscripts , buildTSSRegions , <EOL> importTranscripts , <EOL> importRegionsOfInterest , exportRegionsOfInterest , <EOL> importSNPsOfInterest , <EOL> exportReferenceAsBed , <EOL> importUCSCEncodeTracks , exportUCSCEncodeTracks , <EOL> importTranscriptInformation , <EOL> importGeneInformation , importGeneStats , <EOL> importGWAS , <EOL> buildAnnotatorGeneTerritories , <EOL> importRegionsOfInterestGenes ) <EOL> def prepare ( ) : <EOL> pass <EOL> @ follows ( buildIntervals , importReferenceIntervals ) <EOL> def intervals ( ) : <EOL> pass <EOL> @ follows ( buildBigwig , viewIntervals , viewBigwig , viewExpressionDifferences ) <EOL> def export ( ) : <EOL> pass <EOL> @ follows ( importAffymetrixAnnotation , buildProbeset2Transcript , importProbeset2Transcript , <EOL> buildExpressionTracks , importExpressionTracks , <EOL> importExpressionProbesets , importExpressionMap , <EOL> buildExpressionCorrelation , importExpressionCorrelation , <EOL> importExpressionDifferences , exportDifferentiallyExpressedGenes , <EOL> runExpressionGOSlim , runExpressionGO , <EOL> importExpressionGOSlim , importExpressionGO , <EOL> ) <EOL> def expression ( ) : <EOL> '''<STR_LIT>''' <EOL> pass <EOL> @ follows ( makeMotifs , <EOL> exportMotifSequences , <EOL> runMEME , <EOL> runTomTom , importTomTom , <EOL> filterMotifs , <EOL> runGLAM2 , <EOL> exportMotifControlSequences , <EOL> runBioProspector ) <EOL> def build_motifs ( ) : <EOL> pass <EOL> @ follows ( runMAST , importMAST , <EOL> runGLAM2SCAN , importGLAM2SCAN ) <EOL> def find_motifs ( ) : <EOL> pass <EOL> @ follows ( runNubiscanNR , runNubiscanVDR , <EOL> runNubiscanNRShuffled , runNubiscanVDRShuffled , <EOL> runNubiscanNRShifted , runNubiscanVDRShifted , <EOL> importNubiscan ) <EOL> def run_nubiscan ( ) : <EOL> pass <EOL> @ follows ( importCorrelation , <EOL> importOverlap , <EOL> importUCSCOverlap , <EOL> reproducibility ) <EOL> def correlation ( ) : <EOL> pass <EOL> @ follows ( annotateIntervals , importAnnotations , <EOL> annotateTSS , importTSS , <EOL> annotateRepeats , importRepeats , <EOL> annotateTSSIntervalAssociations , importTSSIntervalAssociations , <EOL> annotateTSSIntervalDistance , importTSSIntervalDistance , <EOL> annotateTracks , importTracks , <EOL> buildIntervalCounts , importIntervalCounts , <EOL> buildSNPCoverage , importSNPCoverage , <EOL> importOverlapRegions , <EOL> importOverlapRegionsOfInterestEnsembl , ) <EOL> def annotate ( ) : <EOL> pass <EOL> @ follows ( buildAnnotatorGC , <EOL> makeAnnotatorDistance , <EOL> makeAnnotatorArchitecture , importAnnotatorArchitecture , <EOL> makeAnnotatorTracks , importAnnotatorTracks , <EOL> makeAnnotatorPromotorsGO , importAnnotatorPromotorsGO , <EOL> makeAnnotatorPromotorsGOSlim , importAnnotatorPromotorsGOSlim , <EOL> makeAnnotatorTerritoriesGO , importAnnotatorTerritoriesGO , <EOL> makeAnnotatorTerritoriesGOSlim , importAnnotatorTerritoriesGOSlim , <EOL> annotator_regions , <EOL> annotator_proportion , <EOL> annotator_motifs , <EOL> annotator_roi , <EOL> ) <EOL> def annotator ( ) : <EOL> pass <EOL> @ follows ( prepare , <EOL> intervals , <EOL> build_motifs , <EOL> find_motifs , <EOL> expression , <EOL> correlation , <EOL> annotate , <EOL> annotator <EOL> ) <EOL> def full ( ) : <EOL> pass <EOL> @ files ( ( ( "<STR_LIT:.>" , "<STR_LIT>" ) , ) ) <EOL> def clean ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> patterns = ( "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> ) <EOL> cleaned = P . clean ( patterns , dry_run = False ) <EOL> outf = open ( outfile , "<STR_LIT:a>" ) <EOL> outf . write ( "<STR_LIT>" ) <EOL> for filename , statinfo in cleaned : <EOL> outf . write ( "<STR_LIT:\t>" . join ( map ( str , ( filename , <EOL> statinfo . st_size , <EOL> statinfo . st_atime , <EOL> statinfo . st_mtime , <EOL> statinfo . st_ctime ) ) ) + "<STR_LIT:\n>" ) <EOL> outf . close ( ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> sys . exit ( P . main ( sys . argv ) ) </s>
