<s> """<STR_LIT>""" <EOL> import sys <EOL> import tempfile <EOL> import optparse <EOL> import shutil <EOL> import itertools <EOL> import csv <EOL> import math <EOL> import random <EOL> import re <EOL> import glob <EOL> import os <EOL> import shutil <EOL> import collections <EOL> import gzip <EOL> import sqlite3 <EOL> import pysam <EOL> import CGAT . IndexedFasta as IndexedFasta <EOL> import CGAT . IndexedGenome as IndexedGenome <EOL> import CGAT . FastaIterator as FastaIterator <EOL> import CGAT . Genomics as Genomics <EOL> import CGAT . IOTools as IOTools <EOL> import CGAT . MAST as MAST <EOL> import CGAT . GTF as GTF <EOL> import CGAT . GFF as GFF <EOL> import CGAT . Bed as Bed <EOL> import cStringIO <EOL> import numpy <EOL> import CGAT . Masker as Masker <EOL> import fileinput <EOL> import CGAT . Experiment as E <EOL> import logging as L <EOL> from ruffus import * <EOL> USECLUSTER = True <EOL> import CGAT . Pipeline as P <EOL> P . getParameters ( [ "<STR_LIT>" , ] ) <EOL> PARAMS = P . PARAMS <EOL> @ transform ( "<STR_LIT>" , regex ( r"<STR_LIT>" ) , r"<STR_LIT>" ) <EOL> def convertTxtToGtf ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> track = P . snip ( os . path . basename ( infile ) , "<STR_LIT>" ) <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ transform ( convertTxtToGtf , regex ( r"<STR_LIT>" ) , r"<STR_LIT>" ) <EOL> def getGtfStrandedTranscripts ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> track = P . snip ( os . path . basename ( infile ) , "<STR_LIT>" ) <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ transform ( getGtfStrandedTranscripts , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def getEnsemblCodingGeneset ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> ensembl_genes = PARAMS [ "<STR_LIT>" ] <EOL> ensembl_noncoding = PARAMS [ "<STR_LIT>" ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ transform ( getEnsemblCodingGeneset , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def renameTranscripts ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ transform ( renameTranscripts , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def annotateTranscripts ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> ensembl_genes = PARAMS [ "<STR_LIT>" ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ transform ( annotateTranscripts , regex ( r"<STR_LIT>" ) , r"<STR_LIT>" ) <EOL> def addMissingEnsemblTranscripts ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> ensembl_transcripts = PARAMS [ "<STR_LIT>" ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ follows ( mkdir ( "<STR_LIT>" ) ) <EOL> @ transform ( addMissingEnsemblTranscripts , regex ( r"<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def renameTranscriptToExon ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> statement = """<STR_LIT>""" <EOL> P . run ( ) <EOL> @ transform ( getGtfStrandedTranscripts , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def getNoncodingGeneset ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> ensembl_transcripts = PARAMS [ "<STR_LIT>" ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ transform ( getNoncodingGeneset , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def renameNoncodingTranscripts ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ transform ( renameNoncodingTranscripts , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def annotateNoncodingTranscripts ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> ensembl_noncoding = PARAMS [ "<STR_LIT>" ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ transform ( annotateNoncodingTranscripts , regex ( r"<STR_LIT>" ) , r"<STR_LIT>" ) <EOL> def addMissingNoncodingTranscripts ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> ensembl_noncoding = PARAMS [ "<STR_LIT>" ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ transform ( addMissingEnsemblTranscripts , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def transcriptGtfToTab ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ transform ( addMissingNoncodingTranscripts , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def noncodingGtfToTab ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ follows ( transcriptGtfToTab , noncodingGtfToTab ) <EOL> @ merge ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> def mergeTranscriptInfo ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> inlist = "<STR_LIT:U+0020>" . join ( infiles ) <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ transform ( mergeTranscriptInfo , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def loadTranscripts ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> headers = "<STR_LIT>" <EOL> statement = """<STR_LIT>""" <EOL> P . run ( ) <EOL> @ follows ( mkdir ( "<STR_LIT>" ) ) <EOL> @ transform ( getEnsemblCodingGeneset , regex ( r"<STR_LIT>" ) , r"<STR_LIT>" ) <EOL> def buildGeneIntervals ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> ensembl_genes = PARAMS [ "<STR_LIT>" ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ transform ( buildGeneIntervals , regex ( r"<STR_LIT>" ) , r"<STR_LIT>" ) <EOL> def annotateGeneIntervals ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> ensembl_genes = PARAMS [ "<STR_LIT>" ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ transform ( annotateGeneIntervals , regex ( PARAMS [ '<STR_LIT>' ] ) , PARAMS [ '<STR_LIT>' ] ) <EOL> def buildUpstreamFlankBed ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> window = PARAMS [ "<STR_LIT>" ] <EOL> faidx = PARAMS [ "<STR_LIT>" ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ transform ( annotateGeneIntervals , regex ( PARAMS [ '<STR_LIT>' ] ) , PARAMS [ '<STR_LIT>' ] ) <EOL> def buildDownstreamFlankBed ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> window = PARAMS [ "<STR_LIT>" ] <EOL> faidx = PARAMS [ "<STR_LIT>" ] <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ merge ( ( annotateGeneIntervals , buildUpstreamFlankBed , buildDownstreamFlankBed ) , os . path . join ( "<STR_LIT>" , PARAMS [ '<STR_LIT>' ] ) ) <EOL> def buildIntergenicBed ( infiles , outfile ) : <EOL> '''<STR_LIT>''' <EOL> inlist = "<STR_LIT:U+0020>" . join ( infiles ) <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ transform ( ( annotateGeneIntervals , buildUpstreamFlankBed , buildDownstreamFlankBed , buildIntergenicBed ) , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def convertBedToGtf ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> statement = '''<STR_LIT>''' <EOL> P . run ( ) <EOL> @ transform ( addMissingEnsemblTranscripts , regex ( r"<STR_LIT>" ) , os . path . join ( "<STR_LIT>" , PARAMS [ "<STR_LIT>" ] ) ) <EOL> def buildTranscriptTSS ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> statement = """<STR_LIT>""" <EOL> P . run ( ) <EOL> @ follows ( convertBedToGtf ) <EOL> @ files ( "<STR_LIT>" , os . path . join ( "<STR_LIT>" , PARAMS [ "<STR_LIT>" ] ) ) <EOL> def buildGeneTSS ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> statement = """<STR_LIT>""" <EOL> P . run ( ) <EOL> @ transform ( addMissingEnsemblTranscripts , regex ( r"<STR_LIT>" ) , os . path . join ( "<STR_LIT>" , PARAMS [ "<STR_LIT>" ] ) ) <EOL> def buildGeneTSSInterval ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> statement = """<STR_LIT>""" <EOL> P . run ( ) <EOL> @ transform ( addMissingNoncodingTranscripts , regex ( r"<STR_LIT>" ) , os . path . join ( "<STR_LIT>" , PARAMS [ "<STR_LIT>" ] ) ) <EOL> def buildNoncodingGeneTSS ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> statement = """<STR_LIT>""" <EOL> P . run ( ) <EOL> @ transform ( ( buildTranscriptTSS , buildGeneTSS , buildGeneTSSInterval , buildNoncodingGeneTSS ) , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def ExtendRegion ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> statement = """<STR_LIT>""" <EOL> P . run ( ) <EOL> @ transform ( ( buildTranscriptTSS , buildGeneTSS , buildGeneTSSInterval , buildNoncodingGeneTSS , ExtendRegion ) , suffix ( "<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def convertToGTF ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> statement = """<STR_LIT>""" <EOL> P . run ( ) <EOL> @ files ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> def copyEnsembl ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> ensembl_dir = PARAMS [ "<STR_LIT>" ] <EOL> statement = """<STR_LIT>""" <EOL> P . run ( ) <EOL> @ files ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> def copyEnsemblDb ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> table_list = P . asList ( PARAMS [ "<STR_LIT>" ] ) <EOL> dbhandle = sqlite3 . connect ( PARAMS [ "<STR_LIT>" ] ) <EOL> cc = dbhandle . cursor ( ) <EOL> query = """<STR_LIT>""" % PARAMS [ "<STR_LIT>" ] <EOL> cc . execute ( query ) <EOL> for table in table_list : <EOL> cc = dbhandle . cursor ( ) <EOL> query = """<STR_LIT>""" % ( table , table ) <EOL> print query <EOL> cc . execute ( query ) <EOL> cc . close ( ) <EOL> statement = """<STR_LIT>""" <EOL> P . run ( ) <EOL> @ transform ( loadTranscripts , regex ( r"<STR_LIT>" ) , "<STR_LIT>" ) <EOL> def updateGeneName ( infile , outfile ) : <EOL> '''<STR_LIT>''' <EOL> dbhandle = sqlite3 . connect ( PARAMS [ "<STR_LIT>" ] ) <EOL> cc = dbhandle . cursor ( ) <EOL> query = """<STR_LIT>""" % PARAMS [ "<STR_LIT>" ] <EOL> cc . execute ( query ) <EOL> query = """<STR_LIT>""" <EOL> print query <EOL> cc . execute ( query ) <EOL> cc . close ( ) <EOL> statement = """<STR_LIT>""" <EOL> P . run ( ) <EOL> @ follows ( convertTxtToGtf , getGtfStrandedTranscripts , <EOL> getEnsemblCodingGeneset , renameTranscripts , annotateTranscripts , <EOL> addMissingEnsemblTranscripts , renameTranscriptToExon ) <EOL> def transcripts ( ) : <EOL> '''<STR_LIT>''' <EOL> pass <EOL> @ follows ( getNoncodingGeneset , renameNoncodingTranscripts , <EOL> annotateNoncodingTranscripts , addMissingNoncodingTranscripts ) <EOL> def noncoding ( ) : <EOL> '''<STR_LIT>''' <EOL> pass <EOL> @ follows ( transcriptGtfToTab , noncodingGtfToTab , mergeTranscriptInfo , loadTranscripts ) <EOL> def loadTranscriptInfo ( ) : <EOL> '''<STR_LIT>''' <EOL> pass <EOL> @ follows ( buildGeneIntervals , annotateGeneIntervals , buildUpstreamFlankBed , <EOL> buildDownstreamFlankBed , buildIntergenicBed , convertBedToGtf ) <EOL> def genes ( ) : <EOL> '''<STR_LIT>''' <EOL> pass <EOL> @ follows ( buildTranscriptTSS , buildGeneTSS , buildNoncodingGeneTSS , <EOL> ExtendRegion , convertToGTF ) <EOL> def tss ( ) : <EOL> '''<STR_LIT>''' <EOL> pass <EOL> @ follows ( copyEnsembl , copyEnsemblDb , updateGeneName ) <EOL> def ensembl ( ) : <EOL> '''<STR_LIT>''' <EOL> pass <EOL> @ follows ( transcripts , noncoding , loadTranscriptInfo , genes , tss , ensembl ) <EOL> def full ( ) : <EOL> '''<STR_LIT>''' <EOL> pass <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> sys . exit ( P . main ( sys . argv ) ) </s>
