<s> '''<STR_LIT>''' <EOL> import itertools <EOL> import math <EOL> import sys <EOL> import collections <EOL> import os <EOL> import re <EOL> import string <EOL> import CGAT . GO as GO <EOL> import CGAT . IOTools as IOTools <EOL> import CGAT . Experiment as E <EOL> import numpy <EOL> import networkx <EOL> import networkx . algorithms . traversal . depth_first_search as traversal <EOL> from xml . sax . handler import ContentHandler <EOL> from xml . sax import make_parser <EOL> from networkx import DiGraph <EOL> from networkx import topological_sort <EOL> import cPickle <EOL> import matplotlib . pyplot as plt <EOL> class Tokenizer : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self ) : <EOL> self . pattern_punctuation = re . compile ( <EOL> '<STR_LIT>' ) <EOL> self . pattern_numbers = re . compile ( '<STR_LIT>' ) <EOL> self . pattern_plurals = re . compile ( '<STR_LIT>' ) <EOL> self . pattern_possessives = re . compile ( '<STR_LIT>' ) <EOL> self . pattern_blanks = re . compile ( '<STR_LIT>' ) <EOL> self . pattern_others = re . compile ( <EOL> '<STR_LIT>' ) <EOL> self . pattern_words_and_blanks = re . compile ( '<STR_LIT>' ) <EOL> self . leftchar = { } <EOL> self . leftchar [ '<STR_LIT:U+0020>' ] = <NUM_LIT:0> <EOL> for i in string . ascii_letters : <EOL> self . leftchar [ i ] = <NUM_LIT:0> <EOL> def keepNumChr ( self , words ) : <EOL> newWords = '<STR_LIT>' <EOL> for c in words : <EOL> if c in self . leftchar : <EOL> newWords += c <EOL> return newWords <EOL> def tokenize_word ( self , words ) : <EOL> words = words . lower ( ) <EOL> words = self . pattern_plurals . sub ( '<STR_LIT>' , words ) <EOL> words = self . pattern_possessives . sub ( '<STR_LIT>' , words ) <EOL> words = self . keepNumChr ( words ) <EOL> words = self . pattern_blanks . sub ( '<STR_LIT:U+0020>' , words ) <EOL> return words <EOL> def test ( self , words ) : <EOL> for c in words : <EOL> if c not in self . leftchar : <EOL> self . leftchar [ c ] = <NUM_LIT:1> <EOL> def returnUnexpChar ( self ) : <EOL> reV = [ ] <EOL> for k in self . leftchar . keys ( ) : <EOL> if self . leftchar [ k ] != <NUM_LIT:0> : <EOL> reV += [ k ] <EOL> return reV <EOL> class PorterStemmer : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self ) : <EOL> """<STR_LIT>""" <EOL> self . b = "<STR_LIT>" <EOL> self . k = <NUM_LIT:0> <EOL> self . k0 = <NUM_LIT:0> <EOL> self . j = <NUM_LIT:0> <EOL> def cons ( self , i ) : <EOL> """<STR_LIT>""" <EOL> if self . b [ i ] == '<STR_LIT:a>' or self . b [ i ] == '<STR_LIT:e>' or self . b [ i ] == '<STR_LIT:i>' or self . b [ i ] == '<STR_LIT:o>' or self . b [ i ] == '<STR_LIT:u>' : <EOL> return <NUM_LIT:0> <EOL> if self . b [ i ] == '<STR_LIT:y>' : <EOL> if i == self . k0 : <EOL> return <NUM_LIT:1> <EOL> else : <EOL> return ( not self . cons ( i - <NUM_LIT:1> ) ) <EOL> return <NUM_LIT:1> <EOL> def m ( self ) : <EOL> """<STR_LIT>""" <EOL> n = <NUM_LIT:0> <EOL> i = self . k0 <EOL> while <NUM_LIT:1> : <EOL> if i > self . j : <EOL> return n <EOL> if not self . cons ( i ) : <EOL> break <EOL> i = i + <NUM_LIT:1> <EOL> i = i + <NUM_LIT:1> <EOL> while <NUM_LIT:1> : <EOL> while <NUM_LIT:1> : <EOL> if i > self . j : <EOL> return n <EOL> if self . cons ( i ) : <EOL> break <EOL> i = i + <NUM_LIT:1> <EOL> i = i + <NUM_LIT:1> <EOL> n = n + <NUM_LIT:1> <EOL> while <NUM_LIT:1> : <EOL> if i > self . j : <EOL> return n <EOL> if not self . cons ( i ) : <EOL> break <EOL> i = i + <NUM_LIT:1> <EOL> i = i + <NUM_LIT:1> <EOL> def vowelinstem ( self ) : <EOL> """<STR_LIT>""" <EOL> for i in range ( self . k0 , self . j + <NUM_LIT:1> ) : <EOL> if not self . cons ( i ) : <EOL> return <NUM_LIT:1> <EOL> return <NUM_LIT:0> <EOL> def doublec ( self , j ) : <EOL> """<STR_LIT>""" <EOL> if j < ( self . k0 + <NUM_LIT:1> ) : <EOL> return <NUM_LIT:0> <EOL> if ( self . b [ j ] != self . b [ j - <NUM_LIT:1> ] ) : <EOL> return <NUM_LIT:0> <EOL> return self . cons ( j ) <EOL> def cvc ( self , i ) : <EOL> """<STR_LIT>""" <EOL> if i < ( self . k0 + <NUM_LIT:2> ) or not self . cons ( i ) or self . cons ( i - <NUM_LIT:1> ) or not self . cons ( i - <NUM_LIT:2> ) : <EOL> return <NUM_LIT:0> <EOL> ch = self . b [ i ] <EOL> if ch == '<STR_LIT:w>' or ch == '<STR_LIT:x>' or ch == '<STR_LIT:y>' : <EOL> return <NUM_LIT:0> <EOL> return <NUM_LIT:1> <EOL> def ends ( self , s ) : <EOL> """<STR_LIT>""" <EOL> length = len ( s ) <EOL> if s [ length - <NUM_LIT:1> ] != self . b [ self . k ] : <EOL> return <NUM_LIT:0> <EOL> if length > ( self . k - self . k0 + <NUM_LIT:1> ) : <EOL> return <NUM_LIT:0> <EOL> if self . b [ self . k - length + <NUM_LIT:1> : self . k + <NUM_LIT:1> ] != s : <EOL> return <NUM_LIT:0> <EOL> self . j = self . k - length <EOL> return <NUM_LIT:1> <EOL> def setto ( self , s ) : <EOL> """<STR_LIT>""" <EOL> length = len ( s ) <EOL> self . b = self . b [ : self . j + <NUM_LIT:1> ] + s + self . b [ self . j + length + <NUM_LIT:1> : ] <EOL> self . k = self . j + length <EOL> def r ( self , s ) : <EOL> """<STR_LIT>""" <EOL> if self . m ( ) > <NUM_LIT:0> : <EOL> self . setto ( s ) <EOL> def step1ab ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . b [ self . k ] == '<STR_LIT:s>' : <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> self . k = self . k - <NUM_LIT:2> <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> self . setto ( "<STR_LIT:i>" ) <EOL> elif self . b [ self . k - <NUM_LIT:1> ] != '<STR_LIT:s>' : <EOL> self . k = self . k - <NUM_LIT:1> <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> if self . m ( ) > <NUM_LIT:0> : <EOL> self . k = self . k - <NUM_LIT:1> <EOL> elif ( self . ends ( "<STR_LIT>" ) or self . ends ( "<STR_LIT>" ) ) and self . vowelinstem ( ) : <EOL> self . k = self . j <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> self . setto ( "<STR_LIT>" ) <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> self . setto ( "<STR_LIT>" ) <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> self . setto ( "<STR_LIT>" ) <EOL> elif self . doublec ( self . k ) : <EOL> self . k = self . k - <NUM_LIT:1> <EOL> ch = self . b [ self . k ] <EOL> if ch == '<STR_LIT:l>' or ch == '<STR_LIT:s>' or ch == '<STR_LIT:z>' : <EOL> self . k = self . k + <NUM_LIT:1> <EOL> elif ( self . m ( ) == <NUM_LIT:1> and self . cvc ( self . k ) ) : <EOL> self . setto ( "<STR_LIT:e>" ) <EOL> def step1c ( self ) : <EOL> """<STR_LIT>""" <EOL> if ( self . ends ( "<STR_LIT:y>" ) and self . vowelinstem ( ) ) : <EOL> self . b = self . b [ : self . k ] + '<STR_LIT:i>' + self . b [ self . k + <NUM_LIT:1> : ] <EOL> def step2 ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . b [ self . k - <NUM_LIT:1> ] == '<STR_LIT:a>' : <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> elif self . b [ self . k - <NUM_LIT:1> ] == '<STR_LIT:c>' : <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> elif self . b [ self . k - <NUM_LIT:1> ] == '<STR_LIT:e>' : <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> elif self . b [ self . k - <NUM_LIT:1> ] == '<STR_LIT:l>' : <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT:e>" ) <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> elif self . b [ self . k - <NUM_LIT:1> ] == '<STR_LIT:o>' : <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> elif self . b [ self . k - <NUM_LIT:1> ] == '<STR_LIT:s>' : <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> elif self . b [ self . k - <NUM_LIT:1> ] == '<STR_LIT:t>' : <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> elif self . b [ self . k - <NUM_LIT:1> ] == '<STR_LIT:g>' : <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> def step3 ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . b [ self . k ] == '<STR_LIT:e>' : <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> elif self . b [ self . k ] == '<STR_LIT:i>' : <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> elif self . b [ self . k ] == '<STR_LIT:l>' : <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> elif self . b [ self . k ] == '<STR_LIT:s>' : <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> self . r ( "<STR_LIT>" ) <EOL> def step4 ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . b [ self . k - <NUM_LIT:1> ] == '<STR_LIT:a>' : <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> pass <EOL> else : <EOL> return <EOL> elif self . b [ self . k - <NUM_LIT:1> ] == '<STR_LIT:c>' : <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> pass <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> pass <EOL> else : <EOL> return <EOL> elif self . b [ self . k - <NUM_LIT:1> ] == '<STR_LIT:e>' : <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> pass <EOL> else : <EOL> return <EOL> elif self . b [ self . k - <NUM_LIT:1> ] == '<STR_LIT:i>' : <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> pass <EOL> else : <EOL> return <EOL> elif self . b [ self . k - <NUM_LIT:1> ] == '<STR_LIT:l>' : <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> pass <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> pass <EOL> else : <EOL> return <EOL> elif self . b [ self . k - <NUM_LIT:1> ] == '<STR_LIT:n>' : <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> pass <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> pass <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> pass <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> pass <EOL> else : <EOL> return <EOL> elif self . b [ self . k - <NUM_LIT:1> ] == '<STR_LIT:o>' : <EOL> if self . ends ( "<STR_LIT>" ) and ( self . b [ self . j ] == <EOL> '<STR_LIT:s>' or self . b [ self . j ] == '<STR_LIT:t>' ) : <EOL> pass <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> pass <EOL> else : <EOL> return <EOL> elif self . b [ self . k - <NUM_LIT:1> ] == '<STR_LIT:s>' : <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> pass <EOL> else : <EOL> return <EOL> elif self . b [ self . k - <NUM_LIT:1> ] == '<STR_LIT:t>' : <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> pass <EOL> elif self . ends ( "<STR_LIT>" ) : <EOL> pass <EOL> else : <EOL> return <EOL> elif self . b [ self . k - <NUM_LIT:1> ] == '<STR_LIT:u>' : <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> pass <EOL> else : <EOL> return <EOL> elif self . b [ self . k - <NUM_LIT:1> ] == '<STR_LIT:v>' : <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> pass <EOL> else : <EOL> return <EOL> elif self . b [ self . k - <NUM_LIT:1> ] == '<STR_LIT:z>' : <EOL> if self . ends ( "<STR_LIT>" ) : <EOL> pass <EOL> else : <EOL> return <EOL> else : <EOL> return <EOL> if self . m ( ) > <NUM_LIT:1> : <EOL> self . k = self . j <EOL> def step5 ( self ) : <EOL> """<STR_LIT>""" <EOL> self . j = self . k <EOL> if self . b [ self . k ] == '<STR_LIT:e>' : <EOL> a = self . m ( ) <EOL> if a > <NUM_LIT:1> or ( a == <NUM_LIT:1> and not self . cvc ( self . k - <NUM_LIT:1> ) ) : <EOL> self . k = self . k - <NUM_LIT:1> <EOL> if self . b [ self . k ] == '<STR_LIT:l>' and self . doublec ( self . k ) and self . m ( ) > <NUM_LIT:1> : <EOL> self . k = self . k - <NUM_LIT:1> <EOL> def stem ( self , p , i , j ) : <EOL> """<STR_LIT>""" <EOL> self . b = p <EOL> self . k = j <EOL> self . k0 = i <EOL> if self . k <= self . k0 + <NUM_LIT:1> : <EOL> return self . b <EOL> self . step1ab ( ) <EOL> self . step1c ( ) <EOL> self . step2 ( ) <EOL> self . step3 ( ) <EOL> self . step4 ( ) <EOL> self . step5 ( ) <EOL> return self . b [ self . k0 : self . k + <NUM_LIT:1> ] <EOL> class GOOboXmlHandler ( ContentHandler ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , goGraph ) : <EOL> self . namespace = goGraph . getNameSpace ( ) <EOL> self . graph = goGraph <EOL> self . inTypedef = False <EOL> self . goNode = GONode ( ) <EOL> self . obsolete = False <EOL> self . relationship_type = None <EOL> self . synonyms = { } <EOL> self . obsoletes = set ( ) <EOL> def startElement ( self , name , attributes ) : <EOL> self . cdata = "<STR_LIT>" <EOL> if name == "<STR_LIT>" : <EOL> self . goid = None <EOL> self . name = None <EOL> self . description = None <EOL> self . obsolete = False <EOL> self . elementNamespace = None <EOL> self . parents = [ ] <EOL> self . goNode = GONode ( ) <EOL> elif name == "<STR_LIT>" : <EOL> self . inTypedef = True <EOL> def endElement ( self , name ) : <EOL> if name == "<STR_LIT:id>" : <EOL> self . goid = self . cdata . strip ( ) <EOL> self . goNode . setGOID ( self . goid ) <EOL> elif name == "<STR_LIT>" : <EOL> self . elementNamespace = self . cdata . strip ( ) <EOL> self . goNode . setNamespace ( self . elementNamespace ) <EOL> elif name == "<STR_LIT>" and not self . inTypedef : <EOL> self . parents . append ( self . cdata . strip ( ) ) <EOL> self . goNode . setParents ( self . parents ) <EOL> elif name == "<STR_LIT:type>" : <EOL> self . relationship_type = self . cdata . strip ( ) <EOL> elif name == "<STR_LIT>" and not self . inTypedef : <EOL> if self . relationship_type == "<STR_LIT>" : <EOL> self . parents . append ( self . cdata . strip ( ) ) <EOL> self . goNode . setParents ( self . parents ) <EOL> elif name == "<STR_LIT>" and self . cdata . strip ( ) == "<STR_LIT:1>" : <EOL> self . obsolete = True <EOL> self . goNode . setObsolete ( self . obsolete ) <EOL> elif name == "<STR_LIT>" : <EOL> self . inTypedef = False <EOL> elif name == "<STR_LIT>" : <EOL> self . synonyms [ self . cdata . strip ( ) ] = self . goid <EOL> elif ( name == "<STR_LIT>" and <EOL> not self . obsolete and <EOL> self . namespace == self . elementNamespace ) : <EOL> '''<STR_LIT>''' <EOL> self . graph . add_node ( self . goid , data = self . goNode ) <EOL> for parent in self . parents : <EOL> '''<STR_LIT>''' <EOL> if parent not in self . graph : <EOL> self . graph . add_node ( parent , data = GONode ( parent ) ) <EOL> self . graph . add_edge ( <EOL> parent , self . goid , relationship = "<STR_LIT>" ) <EOL> elif ( name == "<STR_LIT:name>" and <EOL> not self . obsolete and <EOL> self . goid is not None and self . goNode . name is None ) : <EOL> self . goNode . setName ( self . cdata . strip ( ) ) <EOL> elif ( name == "<STR_LIT>" and <EOL> not self . obsolete and <EOL> self . goid is not None ) : <EOL> self . goNode . setDescription ( self . cdata . strip ( ) ) <EOL> elif ( name == "<STR_LIT>" and <EOL> self . obsolete and <EOL> self . namespace == self . elementNamespace ) : <EOL> self . obsoletes . add ( self . goid ) <EOL> def characters ( self , data ) : <EOL> self . cdata += data <EOL> class GONode ( ) : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , goid = None , namespace = None , parents = None , obsolete = False , <EOL> name = None , description = None , genes = set ( ) , propGenes = set ( ) , <EOL> pmids = set ( ) , propPmids = set ( ) , wordVector = dict ( ) , <EOL> descendantCount = None , mergedGenes = set ( ) , mergedPmids = set ( ) , <EOL> mergedCount = <NUM_LIT:0> , infoLoss = <NUM_LIT:0> ) : <EOL> self . goid = goid <EOL> self . namespace = namespace <EOL> self . parents = parents <EOL> self . obsolete = obsolete <EOL> self . name = name <EOL> self . description = description <EOL> self . genes = genes <EOL> self . propGenes = propGenes <EOL> self . pmids = pmids <EOL> self . propPmids = propPmids <EOL> self . wordVector = wordVector <EOL> self . descendantCount = descendantCount <EOL> self . mergedGenes = mergedGenes <EOL> self . mergedPmids = mergedPmids <EOL> self . mergedCount = mergedCount <EOL> self . infoLoss = infoLoss <EOL> def setGOID ( self , goid ) : <EOL> self . goid = goid <EOL> def getGOID ( self ) : <EOL> return self . goid <EOL> def setNamespace ( self , namespace ) : <EOL> self . namespace = namespace <EOL> def getNamespace ( self ) : <EOL> return self . namespace <EOL> def setParents ( self , parents ) : <EOL> self . parents = parents <EOL> def getParents ( self ) : <EOL> return self . parents <EOL> def setObsolete ( self , obsolete ) : <EOL> self . obsolete = obsolete <EOL> def getObsolete ( self ) : <EOL> return self . obsolete <EOL> def setName ( self , name ) : <EOL> self . name = name <EOL> def getName ( self ) : <EOL> return self . name <EOL> def setDescription ( self , description ) : <EOL> self . description = description <EOL> def getDescription ( self ) : <EOL> return self . description <EOL> def setPMIDs ( self , pmids ) : <EOL> self . pmids = pmids <EOL> def getPMIDs ( self ) : <EOL> return self . pmids <EOL> def addPMIDs ( self , pmid ) : <EOL> self . pmids = self . pmids . union ( pmid ) <EOL> def setPropagatedPMIDs ( self , propPmids ) : <EOL> self . propPmids = propPmids <EOL> def getPropagatedPMIDs ( self ) : <EOL> return self . propPmids <EOL> def addPropagatedPMIDs ( self , pmids ) : <EOL> self . propPmids = self . propPmids . union ( pmids ) <EOL> def setGenes ( self , genes ) : <EOL> self . genes = genes <EOL> def getGenes ( self ) : <EOL> return self . genes <EOL> def addGenes ( self , genes ) : <EOL> self . genes = self . genes . union ( genes ) <EOL> def setPropagatedGenes ( self , propGenes ) : <EOL> self . propGenes = propGenes <EOL> def getPropagatedGenes ( self ) : <EOL> return self . propGenes <EOL> def addPropagatedGenes ( self , genes ) : <EOL> self . propGenes = self . propGenes . union ( genes ) <EOL> def calculateWordVector ( self , corpus , tokenizer = Tokenizer ( ) . tokenize_word , <EOL> stemmer = PorterStemmer ( ) . stem , <EOL> stopwords = [ ] , pmids = None ) : <EOL> wordVector = { } <EOL> if not corpus : <EOL> print "<STR_LIT>" "<STR_LIT>" <EOL> else : <EOL> if not pmids : <EOL> pmids = self . getPropagatedPMIDs ( ) <EOL> for pmid , qualifier in pmids : <EOL> if pmid in corpus . docs and "<STR_LIT>" not in qualifier : <EOL> pmidWordVector = corpus . docs [ pmid ] . getWordVector ( <EOL> tokenizer , stemmer , stopwords ) <EOL> for word in pmidWordVector : <EOL> if word in wordVector : <EOL> wordVector [ word ] += pmidWordVector [ word ] <EOL> else : <EOL> wordVector [ word ] = pmidWordVector [ word ] <EOL> self . wordVector = wordVector <EOL> def getWordVector ( self , corpus = None , stopwords = [ ] ) : <EOL> if len ( self . wordVector ) == <NUM_LIT:0> : <EOL> if len ( self . getPropagatedPMIDs ( ) ) > <NUM_LIT:0> : <EOL> self . calculateWordVector ( corpus , stopwords = stopwords ) <EOL> return self . wordVector <EOL> def getDescendantCount ( self ) : <EOL> return self . descendantCount <EOL> def setMergedGenes ( self , mergedGenes ) : <EOL> self . mergedGenes = mergedGenes <EOL> def getMergedGenes ( self ) : <EOL> return self . mergedGenes <EOL> def addMergedGenes ( self , mergedGenes ) : <EOL> self . mergedGenes = self . mergedGenes . union ( mergedGenes ) <EOL> def setMergedPMIDs ( self , mergedPmids ) : <EOL> self . mergedPmids = mergedPmids <EOL> def getMergedPMIDs ( self ) : <EOL> return self . mergedPmids <EOL> def addMergedPMIDs ( self , mergedPmids ) : <EOL> self . mergedPmids = self . mergedPmids . union ( mergedPmids ) <EOL> def setMergedCount ( self , mergedCount ) : <EOL> self . mergedCount = mergedCount <EOL> def getMergedCount ( self ) : <EOL> return self . mergedCount <EOL> def addMergedCount ( self , mergedCount = <NUM_LIT:1> ) : <EOL> self . mergedCount += mergedCount <EOL> def setInfoLoss ( self , infoLoss ) : <EOL> self . infoLoss = infoLoss <EOL> def getInfoLoss ( self ) : <EOL> return self . infoLoss <EOL> def addInfoLoss ( self , infoLoss ) : <EOL> self . infoLoss += infoLoss <EOL> class GOGraph ( networkx . DiGraph ) : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , namespace = None , GOOboXmlFileName = None ) : <EOL> DiGraph . __init__ ( self ) <EOL> self . namespace = namespace <EOL> if GOOboXmlFileName is not None : <EOL> self . parseOboXml ( GOOboXmlFileName ) <EOL> def parseOboXml ( self , GOOboXmlFileName ) : <EOL> parser = make_parser ( ) <EOL> handler = GOOboXmlHandler ( self ) <EOL> parser . setContentHandler ( handler ) <EOL> f = IOTools . openFile ( GOOboXmlFileName , '<STR_LIT:r>' ) <EOL> parser . parse ( f ) <EOL> f . close ( ) <EOL> self . synonyms = handler . synonyms <EOL> self . obsoletes = handler . obsoletes <EOL> def getLevel ( self , goid ) : <EOL> parents = self . predecessors ( goid ) <EOL> if len ( parents ) == <NUM_LIT:0> : <EOL> return <NUM_LIT:0> <EOL> found = False <EOL> level = <NUM_LIT:1> <EOL> while not found : <EOL> nextDepth = list ( ) <EOL> for i in parents : <EOL> pred = self . predecessors ( i ) <EOL> if len ( pred ) == <NUM_LIT:0> : <EOL> found = True <EOL> return level <EOL> else : <EOL> nextDepth = list ( set ( nextDepth ) | set ( pred ) ) <EOL> parents = nextDepth <EOL> level = level + <NUM_LIT:1> <EOL> def getNameSpace ( self ) : <EOL> return self . namespace <EOL> def savePickle ( self , filename = "<STR_LIT>" ) : <EOL> try : <EOL> f = open ( filename , '<STR_LIT:wb>' ) <EOL> cPickle . dump ( self , f , protocol = cPickle . HIGHEST_PROTOCOL ) <EOL> f . close ( ) <EOL> except : <EOL> print "<STR_LIT>" <EOL> @ classmethod <EOL> def loadPickle ( klass , filename = "<STR_LIT>" ) : <EOL> try : <EOL> f = open ( filename , '<STR_LIT:rb>' ) <EOL> g = cPickle . load ( f ) <EOL> return g <EOL> except : <EOL> print "<STR_LIT>" <EOL> return <EOL> def getNodeDescription ( self , goid ) : <EOL> if goid in self . nodes ( ) : <EOL> return self . node [ goid ] [ '<STR_LIT:data>' ] . getDescription ( ) <EOL> else : <EOL> print '<STR_LIT>' <EOL> raise <EOL> def __setNodeDescription ( self , goid , descrip ) : <EOL> if goid in self . nodes ( ) : <EOL> self . node [ goid ] [ '<STR_LIT:data>' ] . setDescription ( descrip ) <EOL> else : <EOL> print '<STR_LIT>' <EOL> raise <EOL> def getNodeNamespace ( self , goid ) : <EOL> if goid in self . nodes ( ) : <EOL> return self . node [ goid ] [ '<STR_LIT:data>' ] . getNamespace ( ) <EOL> else : <EOL> print '<STR_LIT>' <EOL> raise <EOL> def __setNodeNamespace ( self , goid , namespace ) : <EOL> if goid in self . nodes ( ) : <EOL> self . node [ goid ] [ '<STR_LIT>' ] = namespace <EOL> else : <EOL> print '<STR_LIT>' <EOL> raise <EOL> def calcDescendantCount ( self ) : <EOL> nodes = dict ( ) <EOL> sortedNodes = topological_sort ( self ) <EOL> sortedNodes . reverse ( ) <EOL> for node in sortedNodes : <EOL> if node not in nodes : <EOL> ids = set ( ) <EOL> else : <EOL> ids = nodes [ node ] <EOL> self . node [ node ] [ '<STR_LIT:data>' ] . descendantCount = len ( ids ) <EOL> ids = ids . union ( [ node ] ) <EOL> for parent in self . predecessors ( node ) : <EOL> if parent not in nodes : <EOL> nodes [ parent ] = ids <EOL> else : <EOL> nodes [ parent ] = nodes [ parent ] . union ( ids ) <EOL> def getDescendantCount ( self , goid ) : <EOL> if goid in self : <EOL> count = self . node [ goid ] [ '<STR_LIT:data>' ] . getDescendantCount ( ) <EOL> if not count : <EOL> self . calcDescendantCount ( ) <EOL> return self . node [ goid ] [ '<STR_LIT:data>' ] . getDescendantCount ( ) <EOL> else : <EOL> return count <EOL> def getAncestors ( g , node ) : <EOL> p = g . predecessors ( node ) <EOL> ancestors = set ( p ) <EOL> work = p [ : ] <EOL> while work : <EOL> x = work . pop ( ) <EOL> p = g . predecessors ( x ) <EOL> work . extend ( [ y for y in p if y not in ancestors ] ) <EOL> ancestors . update ( p ) <EOL> return ancestors <EOL> def readPValues ( infile , synonyms = { } ) : <EOL> '''<STR_LIT>''' <EOL> term2pvalue , term2log2fold = { } , collections . defaultdict ( float ) <EOL> def _parse2 ( data ) : <EOL> goid = synonyms . get ( data [ <NUM_LIT:0> ] , data [ <NUM_LIT:0> ] ) <EOL> term2pvalue [ goid ] = float ( data [ <NUM_LIT:1> ] ) <EOL> def _parse3 ( data ) : <EOL> goid = synonyms . get ( data [ <NUM_LIT:0> ] , data [ <NUM_LIT:0> ] ) <EOL> term2pvalue [ goid ] = float ( data [ <NUM_LIT:1> ] ) <EOL> term2log2fold [ goid ] = math . log ( float ( data [ <NUM_LIT:2> ] ) , <NUM_LIT:2> ) <EOL> def _parseGO ( data ) : <EOL> '''<STR_LIT>''' <EOL> if len ( data ) < <NUM_LIT> : <EOL> return <EOL> goid = synonyms . get ( data [ <NUM_LIT:11> ] , data [ <NUM_LIT:11> ] ) <EOL> term2pvalue [ goid ] = float ( data [ <NUM_LIT:8> ] ) <EOL> term2log2fold [ goid ] = math . log ( float ( data [ <NUM_LIT:7> ] ) + <NUM_LIT> , <NUM_LIT:2> ) <EOL> parser = None <EOL> for line in infile : <EOL> if line . startswith ( "<STR_LIT:code>" ) : <EOL> parser = _parseGO <EOL> continue <EOL> elif line . startswith ( "<STR_LIT>" ) : <EOL> data = line [ : - <NUM_LIT:1> ] . split ( "<STR_LIT:\t>" ) <EOL> if len ( data ) == <NUM_LIT:2> : <EOL> parser = _parse2 <EOL> else : <EOL> parser = _parse3 <EOL> continue <EOL> if parser is None : <EOL> raise ValueError ( <EOL> "<STR_LIT>" % line ) <EOL> parser ( line [ : - <NUM_LIT:1> ] . split ( ) ) <EOL> return term2pvalue , term2log2fold <EOL> def readSimrelMatrix ( infile ) : <EOL> '''<STR_LIT>''' <EOL> matrix , row_headers , col_headers = IOTools . readMatrix ( infile ) <EOL> assert row_headers == col_headers <EOL> return matrix , row_headers <EOL> def readTermFrequencies ( infile ) : <EOL> '''<STR_LIT>''' <EOL> counts , p = { } , { } <EOL> for line in infile : <EOL> if line . startswith ( "<STR_LIT>" ) : <EOL> continue <EOL> goid , f , c = line [ : - <NUM_LIT:1> ] . split ( "<STR_LIT:\t>" ) <EOL> p [ goid ] = float ( f ) <EOL> counts [ goid ] = set ( c . split ( "<STR_LIT:;>" ) ) <EOL> return counts , p <EOL> def readAncestors ( infile ) : <EOL> '''<STR_LIT>''' <EOL> ancestors = { } <EOL> for line in infile : <EOL> if line . startswith ( "<STR_LIT>" ) : <EOL> continue <EOL> goid , anc = line [ : - <NUM_LIT:1> ] . split ( "<STR_LIT:\t>" ) <EOL> ancestors [ goid ] = set ( anc . split ( "<STR_LIT:;>" ) ) <EOL> return ancestors <EOL> def computeAncestors ( graph , nodes ) : <EOL> '''<STR_LIT>''' <EOL> iteration , total = <NUM_LIT:0> , len ( nodes ) <EOL> E . info ( "<STR_LIT>" % len ( nodes ) ) <EOL> ancestors = { } <EOL> for node in nodes : <EOL> a = getAncestors ( graph , node ) <EOL> a = set ( [ x for x in a if x in nodes ] ) <EOL> ancestors [ node ] = a <EOL> iteration += <NUM_LIT:1> <EOL> if iteration % <NUM_LIT> == <NUM_LIT:0> : <EOL> E . debug ( "<STR_LIT>" % <EOL> ( iteration , <NUM_LIT> * iteration / total ) ) <EOL> return ancestors <EOL> def getRoot ( graph ) : <EOL> '''<STR_LIT>''' <EOL> root = graph . nodes ( ) [ <NUM_LIT:0> ] <EOL> p = graph . predecessors ( root ) <EOL> while p != [ ] : <EOL> root = p [ <NUM_LIT:0> ] <EOL> p = graph . predecessors ( root ) <EOL> return root <EOL> def buildFilename ( ontology , section , suffix ) : <EOL> return E . getOutputFile ( "<STR_LIT>" % ( ontology , section , suffix ) ) <EOL> def computeTermFrequencies ( graph , go2genes ) : <EOL> nodes = graph . nodes ( ) <EOL> root = getRoot ( graph ) <EOL> E . info ( "<STR_LIT>" ) <EOL> counts = collections . defaultdict ( set ) <EOL> E . info ( "<STR_LIT>" ) <EOL> for node in traversal . dfs_postorder ( graph , source = root ) : <EOL> counts [ node ] = set ( go2genes [ node ] ) <EOL> for x in graph . successors ( node ) : <EOL> counts [ node ] . update ( counts [ x ] ) <EOL> rf = float ( len ( counts [ root ] ) ) <EOL> p = { } <EOL> for term , v in counts . iteritems ( ) : <EOL> p [ term ] = len ( v ) / rf <EOL> E . info ( "<STR_LIT>" % len ( p ) ) <EOL> for node in nodes : <EOL> pre = graph . predecessors ( node ) <EOL> while pre != [ ] : <EOL> assert p [ pre [ <NUM_LIT:0> ] ] >= p [ node ] <EOL> node = pre [ <NUM_LIT:0> ] <EOL> pre = graph . predecessors ( node ) <EOL> return counts , p <EOL> def buildSimrelMatrix ( go2genes , go2info , ancestors , p ) : <EOL> '''<STR_LIT>''' <EOL> lp = { } <EOL> for term , v in p . iteritems ( ) : <EOL> if v > <NUM_LIT:0> and v < <NUM_LIT:1.0> : <EOL> lp [ term ] = math . log ( v ) <EOL> test_nodes = lp . keys ( ) <EOL> node2index = dict ( [ ( y , x ) for x , y in enumerate ( test_nodes ) ] ) <EOL> E . info ( "<STR_LIT>" % <EOL> ( len ( test_nodes ) , len ( test_nodes ) ) ) <EOL> matrix = numpy . zeros ( ( len ( test_nodes ) , len ( test_nodes ) ) ) <EOL> iteration = <NUM_LIT:0> <EOL> total = len ( test_nodes ) * ( len ( test_nodes ) - <NUM_LIT:1> ) / <NUM_LIT:2> <EOL> for c1 , c2 in itertools . combinations ( test_nodes , <NUM_LIT:2> ) : <EOL> iteration += <NUM_LIT:1> <EOL> if iteration % <NUM_LIT> == <NUM_LIT:0> : <EOL> E . info ( "<STR_LIT>" % <EOL> ( iteration , <NUM_LIT> * iteration / total ) ) <EOL> s1 = ancestors [ c1 ] <EOL> s2 = ancestors [ c2 ] <EOL> common = s1 . intersection ( s2 ) <EOL> i , j = node2index [ c1 ] , node2index [ c2 ] <EOL> f = <NUM_LIT> / ( lp [ c1 ] + lp [ c2 ] ) <EOL> vals = [ lp [ c ] * ( <NUM_LIT:1.0> - p [ c ] ) for c in common if c in lp ] <EOL> if len ( vals ) > <NUM_LIT:0> : <EOL> val = f * min ( vals ) <EOL> assert <NUM_LIT:0> <= val <= <NUM_LIT:1> <EOL> matrix [ i ] [ j ] = val <EOL> matrix [ j ] [ i ] = val <EOL> return matrix , test_nodes <EOL> def clusterSimrelMatrix ( matrix , terms , ancestors , p , <EOL> counts , term2pvalue , go2info , options ) : <EOL> '''<STR_LIT>''' <EOL> E . info ( "<STR_LIT>" % len ( terms ) ) <EOL> term2index = dict ( [ ( y , x ) for x , y in enumerate ( terms ) ] ) <EOL> terms = [ x for x in terms if x in term2pvalue ] <EOL> E . info ( "<STR_LIT>" % len ( terms ) ) <EOL> if len ( terms ) == <NUM_LIT:0> : <EOL> E . warn ( "<STR_LIT>" ) <EOL> return None , [ ] , [ ] <EOL> matrix = matrix [ numpy . array ( [ term2index [ x ] for x in terms ] ) ] <EOL> matrix = matrix [ : , numpy . array ( [ term2index [ x ] for x in terms ] ) ] <EOL> term2index = dict ( [ ( y , x ) for x , y in enumerate ( terms ) ] ) <EOL> clusters = dict ( [ ( x , [ x ] ) for x in terms ] ) <EOL> nrows = len ( terms ) <EOL> counter = E . Counter ( ) <EOL> member_indices = [ ] <EOL> while <NUM_LIT:1> : <EOL> index = numpy . argmax ( matrix ) <EOL> i = index // nrows <EOL> j = index % nrows <EOL> simrel = matrix [ i ] [ j ] <EOL> assert simrel == matrix . max ( ) <EOL> if simrel == <NUM_LIT:0> : <EOL> E . debug ( "<STR_LIT>" ) <EOL> break <EOL> if simrel < options . max_simrel_threshold : <EOL> E . debug ( "<STR_LIT>" % <EOL> ( simrel , options . max_simrel_threshold ) ) <EOL> break <EOL> nclusters = len ( clusters ) <EOL> if nclusters <= options . min_clusters_threshold : <EOL> E . debug ( "<STR_LIT>" % <EOL> ( nclusters , simrel ) ) <EOL> break <EOL> t1 = terms [ i ] <EOL> t2 = terms [ j ] <EOL> E . debug ( "<STR_LIT>" % ( simrel , i , j ) ) <EOL> p1 = term2pvalue [ t1 ] <EOL> p2 = term2pvalue [ t2 ] <EOL> E . debug ( "<STR_LIT>" % <EOL> ( t1 , go2info . get ( t1 , "<STR_LIT>" ) , p1 , p [ t1 ] * <NUM_LIT> ) ) <EOL> E . debug ( "<STR_LIT>" % <EOL> ( t2 , go2info . get ( t2 , "<STR_LIT>" ) , p2 , p [ t2 ] * <NUM_LIT> ) ) <EOL> counter . tested += <NUM_LIT:1> <EOL> if p [ t1 ] > options . min_frequency : <EOL> counter . min_frequency += <NUM_LIT:1> <EOL> rep , mem = t2 , t1 <EOL> elif p [ t2 ] > options . min_frequency : <EOL> counter . min_frequency += <NUM_LIT:1> <EOL> rep , mem = t1 , t2 <EOL> elif term2pvalue [ t1 ] < term2pvalue [ t2 ] : <EOL> counter . min_pvalue += <NUM_LIT:1> <EOL> rep , mem = t1 , t2 <EOL> elif term2pvalue [ t2 ] < term2pvalue [ t1 ] : <EOL> counter . min_pvalue += <NUM_LIT:1> <EOL> rep , mem = t2 , t1 <EOL> elif t1 in ancestors [ t2 ] : <EOL> ovl = float ( len ( counts [ t1 ] . intersection ( counts [ t2 ] ) ) ) <EOL> if ( ovl / len ( counts [ t1 ] ) ) >= options . min_child_threshold : <EOL> counter . ancestor_child += <NUM_LIT:1> <EOL> rep , mem = t2 , t1 <EOL> else : <EOL> counter . ancestor_parent += <NUM_LIT:1> <EOL> rep , mem = t1 , t2 <EOL> elif t2 in ancestors [ t1 ] : <EOL> ovl = float ( len ( counts [ t1 ] . intersection ( counts [ t2 ] ) ) ) <EOL> if ( ovl / len ( counts [ t2 ] ) ) >= options . min_child_threshold : <EOL> counter . ancestor_child += <NUM_LIT:1> <EOL> rep , mem = t1 , t2 <EOL> else : <EOL> counter . ancestor_parent += <NUM_LIT:1> <EOL> rep , mem = t2 , t1 <EOL> else : <EOL> counter . random += <NUM_LIT:1> <EOL> rep , mem = min ( t1 , t2 ) , max ( t1 , t2 ) <EOL> clusters [ rep ] . extend ( clusters [ mem ] ) <EOL> del clusters [ mem ] <EOL> if mem == t1 : <EOL> matrix [ i , : ] = <NUM_LIT:0> <EOL> matrix [ : , i ] = <NUM_LIT:0> <EOL> member_indices . append ( i ) <EOL> else : <EOL> matrix [ j , : ] = <NUM_LIT:0> <EOL> matrix [ : , j ] = <NUM_LIT:0> <EOL> member_indices . append ( j ) <EOL> E . info ( "<STR_LIT:%s>" % str ( counter ) ) <EOL> E . info ( "<STR_LIT>" % len ( clusters ) ) <EOL> member_indices = set ( member_indices ) <EOL> rep_indices = [ x for x in xrange ( nrows ) if x not in member_indices ] <EOL> terms = [ terms [ x ] for x in rep_indices ] <EOL> matrix = matrix [ numpy . array ( [ x for x in rep_indices ] ) ] <EOL> matrix = matrix [ : , numpy . array ( [ x for x in rep_indices ] ) ] <EOL> return matrix , terms , clusters <EOL> @ E . cachedfunction <EOL> def readGOGraph ( filename_obo , namespace ) : <EOL> E . info ( "<STR_LIT>" % <EOL> ( filename_obo , namespace ) ) <EOL> graph = GOGraph ( namespace = namespace , GOOboXmlFileName = filename_obo ) <EOL> return graph <EOL> def main ( argv ) : <EOL> parser = E . OptionParser ( version = "<STR_LIT>" , <EOL> usage = globals ( ) [ "<STR_LIT>" ] ) <EOL> parser . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> type = "<STR_LIT:string>" , <EOL> help = "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> parser . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , type = "<STR_LIT:string>" , <EOL> help = "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> parser . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> type = "<STR_LIT:string>" , <EOL> help = "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> parser . add_option ( "<STR_LIT>" , dest = "<STR_LIT>" , type = "<STR_LIT>" , <EOL> action = "<STR_LIT>" , <EOL> choices = ( <EOL> "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT:all>" ) , <EOL> help = "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> parser . add_option ( "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> type = "<STR_LIT:float>" , <EOL> help = "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> parser . add_option ( "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> type = "<STR_LIT:float>" , <EOL> help = "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> parser . add_option ( "<STR_LIT>" , dest = "<STR_LIT>" , type = "<STR_LIT>" , <EOL> choices = ( "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ) , <EOL> help = "<STR_LIT>" ) <EOL> parser . add_option ( "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> action = "<STR_LIT:store_true>" , <EOL> help = "<STR_LIT>" ) <EOL> parser . set_defaults ( filename_obo = '<STR_LIT>' , <EOL> filename_go = "<STR_LIT>" , <EOL> filename_pvalues = None , <EOL> filename_bg = None , <EOL> filename_matrix = None , <EOL> filename_frequencies = None , <EOL> ontology = [ ] , <EOL> max_simrel_threshold = <NUM_LIT> , <EOL> min_frequency = <NUM_LIT> , <EOL> min_pvalue = <NUM_LIT> , <EOL> min_child_threshold = <NUM_LIT> , <EOL> min_clusters_threshold = <NUM_LIT:0> , <EOL> palette = "<STR_LIT>" , <EOL> reverse_palette = False , <EOL> ) <EOL> options , args = E . Start ( parser , argv , add_output_options = True ) <EOL> map_ontology2namespace = { <EOL> '<STR_LIT>' : "<STR_LIT>" , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' } <EOL> if "<STR_LIT:all>" in options . ontology : <EOL> options . ontology = map_ontology2namespace . keys ( ) <EOL> if options . filename_pvalues is None : <EOL> E . info ( "<STR_LIT>" ) <EOL> all_term2pvalue , all_term2log2fold = readPValues ( options . stdin ) <EOL> else : <EOL> E . info ( "<STR_LIT>" % options . filename_pvalues ) <EOL> with IOTools . openFile ( options . filename_pvalues ) as infile : <EOL> all_term2pvalue , all_term2log2fold = readPValues ( infile ) <EOL> E . info ( "<STR_LIT>" % <EOL> ( len ( all_term2pvalue ) , len ( all_term2log2fold ) ) ) <EOL> for test_ontology in options . ontology : <EOL> E . info ( "<STR_LIT>" % test_ontology ) <EOL> namespace = map_ontology2namespace [ test_ontology ] <EOL> fn = buildFilename ( test_ontology , "<STR_LIT>" , "<STR_LIT>" ) <EOL> if os . path . exists ( fn ) : <EOL> E . info ( "<STR_LIT>" % fn ) <EOL> with IOTools . openFile ( fn ) as infile : <EOL> ancestors = readAncestors ( infile ) <EOL> else : <EOL> graph = readGOGraph ( options . filename_obo , namespace ) <EOL> ancestors = computeAncestors ( graph , graph . nodes ( ) ) <EOL> with IOTools . openFile ( fn , "<STR_LIT:w>" ) as outfile : <EOL> outfile . write ( "<STR_LIT>" ) <EOL> outfile . write ( <EOL> "<STR_LIT>" . join ( [ "<STR_LIT>" % <EOL> ( x , "<STR_LIT:;>" . join ( v ) ) <EOL> for x , v in ancestors . iteritems ( ) ] ) ) <EOL> graph = readGOGraph ( options . filename_obo , namespace ) <EOL> E . info ( "<STR_LIT>" % options . filename_go ) <EOL> all_gene2gos , all_go2infos = GO . ReadGene2GOFromFile ( <EOL> IOTools . openFile ( options . filename_go ) , <EOL> synonyms = graph . synonyms , <EOL> obsolete = graph . obsoletes ) <EOL> synonyms = graph . synonyms <EOL> term2pvalue = dict ( [ ( synonyms . get ( x , x ) , y ) <EOL> for x , y in all_term2pvalue . iteritems ( ) ] ) <EOL> term2log2fold = dict ( [ ( synonyms . get ( x , x ) , y ) <EOL> for x , y in all_term2log2fold . iteritems ( ) ] ) <EOL> if len ( term2pvalue ) == <NUM_LIT:0> : <EOL> E . warn ( "<STR_LIT>" ) <EOL> E . Stop ( ) <EOL> return <EOL> go2info = all_go2infos [ test_ontology ] <EOL> go2info = collections . defaultdict ( <EOL> str , dict ( [ ( x . mGOId , x . mDescription ) for x in go2info . values ( ) ] ) ) <EOL> gene2gos = all_gene2gos [ test_ontology ] <EOL> if options . filename_bg : <EOL> background = IOTools . readList ( <EOL> IOTools . openFile ( options . filename_bg ) ) <EOL> else : <EOL> background = list ( gene2gos . keys ( ) ) <EOL> go2genes = GO . buildGO2Genes ( gene2gos , ancestors = ancestors ) <EOL> fn = buildFilename ( test_ontology , "<STR_LIT>" , "<STR_LIT>" ) <EOL> if os . path . exists ( fn ) : <EOL> E . info ( "<STR_LIT>" % fn ) <EOL> with IOTools . openFile ( fn ) as infile : <EOL> counts , p = readTermFrequencies ( infile ) <EOL> else : <EOL> counts , p = computeTermFrequencies ( graph , go2genes ) <EOL> with IOTools . openFile ( fn , "<STR_LIT:w>" ) as outfile : <EOL> outfile . write ( "<STR_LIT>" ) <EOL> for k in counts . keys ( ) : <EOL> outfile . write ( "<STR_LIT>" % <EOL> ( k , p [ k ] , "<STR_LIT:;>" . join ( counts [ k ] ) ) ) <EOL> fn = buildFilename ( test_ontology , "<STR_LIT>" , "<STR_LIT>" ) <EOL> if os . path . exists ( fn ) : <EOL> E . info ( "<STR_LIT>" % fn ) <EOL> with IOTools . openFile ( fn , "<STR_LIT:r>" ) as infile : <EOL> matrix , terms = readSimrelMatrix ( infile ) <EOL> else : <EOL> matrix , terms = buildSimrelMatrix ( go2genes , go2info , ancestors , p ) <EOL> E . info ( "<STR_LIT>" ) <EOL> with IOTools . openFile ( fn , "<STR_LIT:w>" ) as outfile : <EOL> IOTools . writeMatrix ( outfile , matrix , terms , terms ) <EOL> fn = buildFilename ( test_ontology , "<STR_LIT>" , "<STR_LIT>" ) <EOL> if os . path . exists ( fn ) : <EOL> E . info ( "<STR_LIT>" % fn ) <EOL> with IOTools . openFile ( fn , "<STR_LIT:r>" ) as infile : <EOL> matrix , terms = readSimrelMatrix ( infile ) <EOL> else : <EOL> matrix , terms , clusters = clusterSimrelMatrix ( matrix , terms , <EOL> ancestors , p , counts , <EOL> term2pvalue , go2info , <EOL> options ) <EOL> if matrix is None : <EOL> E . warn ( "<STR_LIT>" % test_ontology ) <EOL> continue <EOL> E . info ( "<STR_LIT>" % <EOL> ( len ( terms ) , len ( terms ) ) ) <EOL> with IOTools . openFile ( fn , "<STR_LIT:w>" ) as outfile : <EOL> IOTools . writeMatrix ( outfile , matrix , terms , terms ) <EOL> fn = buildFilename ( test_ontology , "<STR_LIT>" , "<STR_LIT>" ) <EOL> with IOTools . openFile ( fn , "<STR_LIT:w>" ) as outfile : <EOL> outfile . write ( <EOL> "<STR_LIT:\t>" . join ( ( "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT:description>" ) ) + "<STR_LIT:\n>" ) <EOL> for rep , members in clusters . iteritems ( ) : <EOL> for mem in members : <EOL> outfile . write ( "<STR_LIT>" % <EOL> ( rep , <EOL> mem , <EOL> term2pvalue [ mem ] , <EOL> p [ mem ] , <EOL> go2info . get ( mem , "<STR_LIT>" ) ) ) <EOL> rows = len ( terms ) <EOL> term_graph = networkx . Graph ( ) <EOL> figure = plt . figure ( ) <EOL> term_graph . add_nodes_from ( terms ) <EOL> for i , j in itertools . combinations ( xrange ( rows ) , <NUM_LIT:2> ) : <EOL> w = matrix [ i ] [ j ] <EOL> if w < <NUM_LIT:0.1> : <EOL> continue <EOL> term_graph . add_edge ( terms [ i ] , terms [ j ] , weight = w ) <EOL> layout = networkx . spring_layout ( term_graph ) <EOL> if options . reverse_palette : <EOL> color_scheme = eval ( "<STR_LIT>" % options . palette ) <EOL> else : <EOL> color_scheme = eval ( "<STR_LIT>" % options . palette ) <EOL> labels = dict ( [ ( x , go2info . get ( x , x ) ) for x in terms ] ) <EOL> node_size = [ - <NUM_LIT> * math . log ( term2pvalue [ x ] , <NUM_LIT:10> ) <EOL> for x in term_graph . nodes ( ) ] <EOL> for x in term2pvalue . keys ( ) : <EOL> term2log2fold [ x ] = numpy . random . normal ( <NUM_LIT:0> , <NUM_LIT:5> ) <EOL> node_color = [ term2log2fold [ x ] for x in term_graph . nodes ( ) ] <EOL> networkx . draw_networkx_edges ( term_graph , layout , edge_color = '<STR_LIT>' ) <EOL> vmin = min ( node_color ) <EOL> vmax = max ( node_color ) <EOL> vmin = min ( vmin , - vmax ) <EOL> vmax = max ( vmax , - vmin ) <EOL> networkx . draw_networkx_nodes ( term_graph , layout , <EOL> node_size = node_size , <EOL> node_color = node_color , <EOL> cmap = color_scheme , <EOL> alpha = <NUM_LIT> , <EOL> vmin = vmin , <EOL> vmax = vmax , <EOL> linewidths = <NUM_LIT:0.5> ) <EOL> positions = dict ( [ ( x , numpy . array ( ( y [ <NUM_LIT:0> ] , y [ <NUM_LIT:1> ] - <NUM_LIT> ) ) ) <EOL> for x , y in layout . iteritems ( ) ] ) <EOL> networkx . draw_networkx_labels ( term_graph , <EOL> positions , <EOL> labels = labels , <EOL> font_size = <NUM_LIT:5> , <EOL> alpha = <NUM_LIT:0.5> ) <EOL> gca = figure . gca ( ) <EOL> gca . set_xticklabels ( [ ] , visibile = False ) <EOL> gca . set_yticklabels ( [ ] , visibile = False ) <EOL> fn = buildFilename ( test_ontology , "<STR_LIT>" , "<STR_LIT>" ) <EOL> plt . colorbar ( ) <EOL> plt . savefig ( fn ) <EOL> fn = buildFilename ( test_ontology , "<STR_LIT>" , "<STR_LIT>" ) <EOL> with IOTools . openFile ( fn , "<STR_LIT:w>" ) as outfile : <EOL> outfile . write ( "<STR_LIT>" ) <EOL> for term in terms : <EOL> outfile . write ( "<STR_LIT>" % <EOL> ( term , <EOL> layout [ term ] [ <NUM_LIT:0> ] , <EOL> layout [ term ] [ <NUM_LIT:1> ] , <EOL> term2pvalue [ term ] , <EOL> term2log2fold [ term ] , <EOL> go2info . get ( term , "<STR_LIT>" ) ) ) <EOL> E . Stop ( ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> sys . exit ( main ( sys . argv ) ) </s>
