<s> from . util import * <EOL> from . vectorizer import * <EOL> import platt <EOL> from sklearn . svm import LinearSVC <EOL> from sklearn . linear_model import LogisticRegression <EOL> import numpy as np <EOL> from sklearn . feature_extraction . text import CountVectorizer <EOL> from scipy . sparse import coo_matrix <EOL> import scipy . stats as stats <EOL> from sklearn . cross_validation import StratifiedKFold <EOL> import sklearn . metrics as metrics <EOL> import scipy as sp <EOL> import pprint <EOL> class TooSkewedLabelsException ( Exception ) : <EOL> def __init__ ( self ) : <EOL> pass <EOL> pass <EOL> class DssgBinaryClassifier ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> pass <EOL> def train ( binaryLabeledMessageList ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( "<STR_LIT>" ) <EOL> def predictScore ( self , message ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( "<STR_LIT>" ) <EOL> def predictProba ( self , message ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( "<STR_LIT>" ) <EOL> pass <EOL> class DssgBinaryClassifierMajorityVote ( DssgBinaryClassifier ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> self . _posNegPrbDic = { } <EOL> def train ( self , binaryLabeledMessageList ) : <EOL> nPos = countTruth ( lambda x : x [ <NUM_LIT:1> ] == '<STR_LIT>' , binaryLabeledMessageList ) <EOL> nNeg = countTruth ( lambda x : x [ <NUM_LIT:1> ] == '<STR_LIT>' , binaryLabeledMessageList ) <EOL> assert ( nPos + nNeg == len ( binaryLabeledMessageList ) ) <EOL> posNegPrbDic = { '<STR_LIT>' : float ( <EOL> nPos ) / ( nPos + nNeg ) , <EOL> '<STR_LIT>' : float ( nNeg ) / ( nPos + nNeg ) } <EOL> self . _posNegPrbDic = posNegPrbDic <EOL> pass <EOL> def predictScore ( self , message ) : <EOL> """<STR_LIT>""" <EOL> scoreDic = { } <EOL> scoreDic [ '<STR_LIT>' ] = ( self . _posNegPrbDic [ '<STR_LIT>' ] - <NUM_LIT> ) <EOL> scoreDic [ '<STR_LIT>' ] = ( self . _posNegPrbDic [ '<STR_LIT>' ] - <NUM_LIT> ) <EOL> return self . _posNegPrbDic <EOL> def predictProba ( self , message ) : <EOL> """<STR_LIT>""" <EOL> return self . _posNegPrbDic <EOL> def __repr__ ( self ) : <EOL> return ( <EOL> self . __class__ . __name__ + '<STR_LIT>' % ( <EOL> self . _posNegPrbDic [ '<STR_LIT>' ] , self . _posNegPrbDic [ '<STR_LIT>' ] ) <EOL> ) <EOL> pass <EOL> class DssgBinaryClassifierSVC ( DssgBinaryClassifier ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , vectorizer , balance = False , C = <NUM_LIT:1.0> , tol = <NUM_LIT> ) : <EOL> self . _vectorizer = vectorizer <EOL> self . _balance = balance <EOL> self . _C = C <EOL> self . _tol = tol <EOL> self . _classifier = None <EOL> self . _plattModel = None <EOL> def train ( self , messageList ) : <EOL> vectorizer = self . _vectorizer <EOL> balance = self . _balance <EOL> C = self . _C <EOL> tol = self . _tol <EOL> msgList = map ( lambda x : x [ <NUM_LIT:0> ] , messageList ) <EOL> y = np . array ( map ( lambda x : <NUM_LIT:1> if x [ <NUM_LIT:1> ] == '<STR_LIT>' else <NUM_LIT:0> , messageList ) ) <EOL> X = vectorizer . fitTransform ( msgList ) <EOL> class_weight = '<STR_LIT>' if balance else None <EOL> classifier = LinearSVC ( <EOL> C = C , loss = '<STR_LIT>' , penalty = '<STR_LIT>' , dual = False , tol = tol , <EOL> random_state = <NUM_LIT:0> , class_weight = class_weight ) <EOL> classifier . fit ( X , y ) <EOL> yDeci = classifier . decision_function ( X ) <EOL> yy = [ <NUM_LIT:1> if v == <NUM_LIT:1> else - <NUM_LIT:1> for v in y ] <EOL> [ A , B ] = platt . SigmoidTrain ( yDeci , yy ) <EOL> plattModel = [ A , B ] <EOL> self . _classifier = classifier <EOL> self . _plattModel = plattModel <EOL> pass <EOL> def predictScore ( self , message ) : <EOL> featVec = self . _vectorizer . transform ( [ message ] ) <EOL> df = self . _classifier . decision_function ( featVec ) <EOL> scoreDic = { } <EOL> scoreDic [ '<STR_LIT>' ] = df [ <NUM_LIT:0> ] <EOL> scoreDic [ '<STR_LIT>' ] = - df [ <NUM_LIT:0> ] <EOL> return scoreDic <EOL> def predictProba ( self , message ) : <EOL> featVec = self . _vectorizer . transform ( [ message ] ) <EOL> df = self . _classifier . decision_function ( featVec ) <EOL> prb = platt . SigmoidPredict ( df , self . _plattModel ) <EOL> probaDic = { } <EOL> probaDic [ '<STR_LIT>' ] = prb <EOL> probaDic [ '<STR_LIT>' ] = <NUM_LIT:1> - prb <EOL> return probaDic <EOL> def __repr__ ( self ) : <EOL> return ( <EOL> self . __class__ . __name__ + '<STR_LIT>' % ( <EOL> self . _vectorizer , self . _classifier , self . _balance , self . _C , self . _tol ) <EOL> ) <EOL> pass <EOL> class DssgBinaryClassifierLogisticRegression ( DssgBinaryClassifier ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , vectorizer , balance = False , C = <NUM_LIT:1.0> , tol = <NUM_LIT> ) : <EOL> self . _vectorizer = vectorizer <EOL> self . _balance = balance <EOL> self . _C = C <EOL> self . _tol = tol <EOL> self . _classifier = None <EOL> def train ( self , messageList ) : <EOL> vectorizer = self . _vectorizer <EOL> balance = self . _balance <EOL> C = self . _C <EOL> tol = self . _tol <EOL> msgList = map ( lambda x : x [ <NUM_LIT:0> ] , messageList ) <EOL> y = np . array ( map ( lambda x : <NUM_LIT:1> if x [ <NUM_LIT:1> ] == '<STR_LIT>' else <NUM_LIT:0> , messageList ) ) <EOL> X = vectorizer . fitTransform ( msgList ) <EOL> class_weight = '<STR_LIT>' if balance else None <EOL> classifier = LogisticRegression ( <EOL> C = C , <EOL> penalty = "<STR_LIT>" , <EOL> dual = False , <EOL> tol = tol , <EOL> random_state = <NUM_LIT:0> , <EOL> class_weight = class_weight ) <EOL> classifier . fit ( X , y ) <EOL> self . _classifier = classifier <EOL> pass <EOL> def predictScore ( self , message ) : <EOL> featVec = self . _vectorizer . transform ( [ message ] ) <EOL> df = self . _classifier . decision_function ( featVec ) <EOL> scoreDic = { } <EOL> scoreDic [ '<STR_LIT>' ] = df [ <NUM_LIT:0> ] <EOL> scoreDic [ '<STR_LIT>' ] = - df [ <NUM_LIT:0> ] <EOL> return scoreDic <EOL> def predictProba ( self , message ) : <EOL> featVec = self . _vectorizer . transform ( [ message ] ) <EOL> ret = self . _classifier . predict_proba ( featVec ) <EOL> probaDic = { } <EOL> probaDic [ '<STR_LIT>' ] = ret [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] <EOL> probaDic [ '<STR_LIT>' ] = ret [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] <EOL> return probaDic <EOL> def __repr__ ( self ) : <EOL> return ( <EOL> self . __class__ . __name__ + '<STR_LIT>' % ( <EOL> self . _vectorizer , self . _classifier , self . _balance , self . _C , self . _tol ) <EOL> ) <EOL> pass <EOL> class DssgBinaryClassifierAdaptiveInterpolation ( DssgBinaryClassifier ) : <EOL> """<STR_LIT>""" <EOL> def __repr__ ( self ) : <EOL> return ( <EOL> self . __class__ . __name__ + '<STR_LIT>' % ( <EOL> self . _globalClassifier , self . _localClassifier , self . _alpha ) <EOL> ) <EOL> def __init__ ( self , globalClassifier , untrainedLocalClassifier ) : <EOL> self . _globalClassifier = globalClassifier <EOL> self . _localClassifier = untrainedLocalClassifier <EOL> self . _alpha = None <EOL> self . _trainLog = None <EOL> self . _plattModel = None <EOL> def train ( self , messageList ) : <EOL> globalClassifier = self . _globalClassifier <EOL> localClassifier = self . _localClassifier <EOL> trainLog = "<STR_LIT>" <EOL> nFolds = <NUM_LIT:5> <EOL> msgList = map ( lambda x : x [ <NUM_LIT:0> ] , messageList ) <EOL> y = np . array ( map ( lambda x : <NUM_LIT:1> if x [ <NUM_LIT:1> ] == '<STR_LIT>' else <NUM_LIT:0> , messageList ) ) <EOL> globalDFList = [ ] <EOL> for msg in msgList : <EOL> v = globalClassifier . predictScore ( msg ) <EOL> globalDFList . append ( v [ '<STR_LIT>' ] ) <EOL> tt = tic ( ) <EOL> if ( sum ( y ) < <NUM_LIT:2> ) : <EOL> trainLog += "<STR_LIT>" <EOL> maxAlpha = <NUM_LIT:0> <EOL> else : <EOL> localDFList = [ float ( '<STR_LIT>' ) ] * len ( messageList ) <EOL> folds = StratifiedKFold ( y , nFolds ) <EOL> for trainIdx , testIdx in folds : <EOL> train = [ messageList [ i ] for i in trainIdx ] <EOL> test = [ messageList [ i ] for i in testIdx ] <EOL> tmpClassifier = copy . deepcopy ( localClassifier ) <EOL> tmpClassifier . train ( train ) <EOL> curDFList = [ ] <EOL> for x in test : <EOL> v = tmpClassifier . predictScore ( x [ <NUM_LIT:0> ] ) <EOL> curDFList . append ( v [ '<STR_LIT>' ] ) <EOL> for i in range ( len ( test ) ) : <EOL> localDFList [ testIdx [ i ] ] = curDFList [ i ] <EOL> assert ( countTruth ( lambda x : np . isnan ( x ) , localDFList ) == <NUM_LIT:0> ) <EOL> f1List = [ ] <EOL> alphaList = np . linspace ( <NUM_LIT:0.0> , <NUM_LIT:1.0> , <NUM_LIT> ) <EOL> for alpha in alphaList : <EOL> combinedDFList = [ ( <NUM_LIT:1> - alpha ) * globalDFList [ i ] + alpha * localDFList [ i ] <EOL> for i in range ( len ( globalDFList ) ) ] <EOL> predy = [ <NUM_LIT:1> if v >= <NUM_LIT:0.0> else <NUM_LIT:0> for v in combinedDFList ] <EOL> f1 = metrics . f1_score ( y , predy ) <EOL> f1List . append ( f1 ) <EOL> trainLog += "<STR_LIT>" <EOL> maxF1 = np . max ( f1List ) <EOL> maxIdx = np . where ( np . array ( f1List ) == maxF1 ) <EOL> maxIdx = maxIdx [ <NUM_LIT:0> ] <EOL> assert ( len ( maxIdx ) != <NUM_LIT:0> ) <EOL> candidateMaxAlpha = [ alphaList [ i ] for i in maxIdx ] <EOL> maxAlpha = np . median ( candidateMaxAlpha ) <EOL> trainLog += "<STR_LIT>" % maxF1 <EOL> trainLog += "<STR_LIT>" % str ( <EOL> np . array ( [ alphaList [ i ] for i in maxIdx ] ) ) <EOL> trainLog += "<STR_LIT>" % maxAlpha <EOL> localClassifier . train ( binaryLabeledMessageList ) <EOL> elapsed = toc ( tt ) <EOL> trainLog += "<STR_LIT>" % elapsed <EOL> self . _localClassifier = localClassifier <EOL> self . _alpha = maxAlpha <EOL> self . _trainLog = trainLog <EOL> yDeci = [ self . predictScore ( msg ) [ '<STR_LIT>' ] for msg in messageList ] <EOL> yy = [ <NUM_LIT:1> if v == <NUM_LIT:1> else - <NUM_LIT:1> for v in y ] <EOL> [ A , B ] = platt . SigmoidTrain ( yDeci , yy ) <EOL> plattModel = [ A , B ] <EOL> self . _plattModel = plattModel <EOL> pass <EOL> def predictScore ( self , message ) : <EOL> if ( self . _localClassifier is None ) : <EOL> return self . _globalClassifier . predictScore ( message ) <EOL> else : <EOL> globalDecisionFunction = self . _globalClassifier . predictScore ( <EOL> message ) [ '<STR_LIT>' ] <EOL> localDecisionFunction = self . _localClassifier . predictScore ( <EOL> message ) [ '<STR_LIT>' ] <EOL> df = ( <NUM_LIT:1> - self . _alpha ) * globalDecisionFunction + self . _alpha * localDecisionFunction <EOL> scoreDic = { } <EOL> scoreDic [ '<STR_LIT>' ] = df <EOL> scoreDic [ '<STR_LIT>' ] = - df <EOL> return scoreDic <EOL> def predictProb ( self , message ) : <EOL> if ( self . _localClassifier is None ) : <EOL> return self . _globalClassifier . predictProba ( message ) <EOL> else : <EOL> globalDecisionFunction = self . _globalClassifier . predictScore ( <EOL> message ) [ '<STR_LIT>' ] <EOL> localDecisionFunction = self . _localClassifier . predictScore ( <EOL> message ) [ '<STR_LIT>' ] <EOL> df = ( <NUM_LIT:1> - self . _alpha ) * globalDecisionFunction + self . _alpha * localDecisionFunction <EOL> prb = platt . SigmoidPredict ( df , self . _plattModel ) <EOL> probaDic = { } <EOL> probaDic [ '<STR_LIT>' ] = prb <EOL> probaDic [ '<STR_LIT>' ] = <NUM_LIT:1> - prb <EOL> return probaDic <EOL> pass <EOL> class DssgBinaryClassifierAdaptiveSVC ( DssgBinaryClassifier ) : <EOL> """<STR_LIT>""" <EOL> def __repr__ ( self ) : <EOL> return ( <EOL> self . __class__ . __name__ + '<STR_LIT>' % ( <EOL> self . _globalClassifier , self . _localClassifier , self . _metaClassifier , self . _alpha ) <EOL> ) <EOL> def __init__ ( self , globalClassifier , untrainedLocalClassifier ) : <EOL> self . _globalClassifier = globalClassifier <EOL> self . _localClassifier = untrainedLocalClassifier <EOL> self . _metaClassifier = None <EOL> self . _trainLog = None <EOL> self . _plattModel = None <EOL> def train ( self , binaryLabeledMessageList ) : <EOL> globalClassifier = self . _globalClassifier <EOL> localClassifier = self . _localClassifier <EOL> trainLog = "<STR_LIT>" <EOL> nFolds = <NUM_LIT:5> <EOL> msgList = map ( lambda x : x [ <NUM_LIT:0> ] , binaryLabeledMessageList ) <EOL> y = np . array ( <EOL> map ( lambda x : <NUM_LIT:1> if x [ <NUM_LIT:1> ] == '<STR_LIT>' else <NUM_LIT:0> , <EOL> binaryLabeledMessageList ) ) <EOL> globalDFList = [ ] <EOL> for msg in msgList : <EOL> v = globalClassifier . predictScore ( msg ) <EOL> globalDFList . append ( v [ '<STR_LIT>' ] ) <EOL> tt = tic ( ) <EOL> metaClassifier = None <EOL> if ( sum ( y ) < <NUM_LIT:2> ) : <EOL> trainLog += "<STR_LIT>" <EOL> localClassifier = None <EOL> else : <EOL> localDFList = [ float ( '<STR_LIT>' ) ] * len ( binaryLabeledMessageList ) <EOL> folds = StratifiedKFold ( y , nFolds ) <EOL> for trainIdx , testIdx in folds : <EOL> train = [ binaryLabeledMessageList [ i ] for i in trainIdx ] <EOL> test = [ binaryLabeledMessageList [ i ] for i in testIdx ] <EOL> tmpClassifier = copy . deepcopy ( localClassifier ) <EOL> tmpClassifier . train ( train ) <EOL> curDFList = [ ] <EOL> for x in test : <EOL> v = tmpClassifier . predictScore ( x [ <NUM_LIT:0> ] ) <EOL> curDFList . append ( v [ '<STR_LIT>' ] ) <EOL> for i in range ( len ( test ) ) : <EOL> localDFList [ testIdx [ i ] ] = curDFList [ i ] <EOL> assert ( countTruth ( lambda x : np . isnan ( x ) , localDFList ) == <NUM_LIT:0> ) <EOL> X = np . hstack ( <EOL> [ np . matrix ( globalDFList ) . T , <EOL> np . matrix ( localDFList ) . T ] ) <EOL> metaClassifier = LinearSVC ( <EOL> loss = '<STR_LIT>' , penalty = '<STR_LIT>' , dual = False , tol = <NUM_LIT> , <EOL> random_state = <NUM_LIT:0> ) <EOL> metaClassifier . fit ( X , y ) <EOL> trainLog += "<STR_LIT>" <EOL> coef = metaClassifier . coef_ [ <NUM_LIT:0> ] <EOL> trainLog += "<STR_LIT>" % ( <EOL> coef [ <NUM_LIT:0> ] , coef [ <NUM_LIT:1> ] ) <EOL> trainLog += "<STR_LIT>" % ( metaClassifier . intercept_ ) <EOL> localClassifier . train ( binaryLabeledMessageList ) <EOL> trainLog += "<STR_LIT>" <EOL> elapsed = toc ( tt ) <EOL> trainLog += "<STR_LIT>" % elapsed <EOL> self . _metaClassifier = metaClassifier <EOL> self . _trainLog = trainLog <EOL> yDeci = [ self . predictScore ( msg ) [ '<STR_LIT>' ] <EOL> for msg in binaryLabeledMessageList ] <EOL> yy = [ <NUM_LIT:1> if v == <NUM_LIT:1> else - <NUM_LIT:1> for v in y ] <EOL> [ A , B ] = platt . SigmoidTrain ( yDeci , yy ) <EOL> plattModel = [ A , B ] <EOL> self . _plattModel = plattModel <EOL> pass <EOL> def predictScore ( self , message ) : <EOL> if ( self . _localClassifier is None ) : <EOL> return self . _globalClassifier . predictScore ( message ) <EOL> else : <EOL> globalDecisionFunction = self . _globalClassifier . predictScore ( <EOL> message ) [ '<STR_LIT>' ] <EOL> localDecisionFunction = self . _localClassifier . predictScore ( <EOL> message ) [ '<STR_LIT>' ] <EOL> testX = np . matrix ( [ globalDecisionFunction , localDecisionFunction ] ) <EOL> df = self . _metaClassifier . decision_function ( testX ) <EOL> scoreDic = { } <EOL> scoreDic [ '<STR_LIT>' ] = df <EOL> scoreDic [ '<STR_LIT>' ] = - df <EOL> return scoreDic <EOL> def predictProb ( self , message ) : <EOL> if ( self . _localClassifier is None ) : <EOL> return self . _globalClassifier . predictProba ( message ) <EOL> else : <EOL> globalDecisionFunction = self . _globalClassifier . predictScore ( <EOL> message ) [ '<STR_LIT>' ] <EOL> localDecisionFunction = self . _localClassifier . predictScore ( <EOL> message ) [ '<STR_LIT>' ] <EOL> testX = np . matrix ( [ globalDecisionFunction , localDecisionFunction ] ) <EOL> df = self . _metaClassifier . decision_function ( testX ) <EOL> prb = platt . SigmoidPredict ( df , self . _plattModel ) <EOL> probaDic = { } <EOL> probaDic [ '<STR_LIT>' ] = prb <EOL> probaDic [ '<STR_LIT>' ] = <NUM_LIT:1> - prb <EOL> return probaDic <EOL> pass <EOL> class DssgCategoryClassifier ( object ) : <EOL> """<STR_LIT>""" <EOL> _classifierDic = { } <EOL> _categoryList = [ ] <EOL> _trainStats = { } <EOL> def getTrainStats ( self ) : <EOL> return self . _trainStats <EOL> def __init__ ( self , <EOL> untrainedBinaryClassifier , <EOL> untrainedBinaryClassifierDic = { } , <EOL> doCV = True , <EOL> verbose = True , <EOL> ) : <EOL> if ( len ( untrainedBinaryClassifierDic ) != <NUM_LIT:0> ) : <EOL> assert untrainedBinaryClassifier is None <EOL> self . _binaryClassifier = untrainedBinaryClassifier <EOL> self . _binaryClassifierDic = untrainedBinaryClassifierDic <EOL> self . _doCV = doCV <EOL> self . _verbose = verbose <EOL> self . _classifierDic = { } <EOL> self . _categoryList = [ ] <EOL> self . _trainStats = { } <EOL> pass <EOL> def __repr__ ( self ) : <EOL> return ( <EOL> self . __class__ . __name__ + '<STR_LIT>' % ( <EOL> self . _categoryList , pprint . pformat ( self . _classifierDic ) ) <EOL> ) <EOL> def train ( self , messageList ) : <EOL> """<STR_LIT>""" <EOL> binaryClassifierDic = self . _binaryClassifierDic <EOL> doCV = self . _doCV <EOL> verbose = self . _verbose <EOL> logger = logging . getLogger ( __name__ ) <EOL> ch = logging . StreamHandler ( ) <EOL> logger . setLevel ( logging . INFO if verbose else logging . WARNING ) <EOL> logger . info ( <EOL> '<STR_LIT>' , <EOL> len ( binaryClassifierDic ) ) <EOL> classifierDic = { } <EOL> categorySet = set ( ) <EOL> for msg in messageList : <EOL> categorySet . update ( msg [ '<STR_LIT>' ] ) <EOL> categorySet . update ( binaryClassifierDic . keys ( ) ) <EOL> categoryList = sorted ( list ( categorySet ) ) <EOL> trainStats = { } <EOL> totNTP = <NUM_LIT:0> <EOL> totNFP = <NUM_LIT:0> <EOL> totNTN = <NUM_LIT:0> <EOL> totNFN = <NUM_LIT:0> <EOL> for cat in categoryList : <EOL> logger . info ( '<STR_LIT>' , cat ) <EOL> if ( len ( binaryClassifierDic ) != <NUM_LIT:0> ) : <EOL> binaryClassifier = binaryClassifierDic [ cat ] <EOL> else : <EOL> binaryClassifier = copy . deepcopy ( self . _binaryClassifier ) <EOL> dset = [ ] <EOL> for msg in messageList : <EOL> binaryLabel = '<STR_LIT>' if cat in msg [ '<STR_LIT>' ] else '<STR_LIT>' <EOL> dset . append ( ( msg , binaryLabel ) ) <EOL> nPos = countTruth ( lambda x : x [ <NUM_LIT:1> ] == '<STR_LIT>' , dset ) <EOL> nNeg = countTruth ( lambda x : x [ <NUM_LIT:1> ] == '<STR_LIT>' , dset ) <EOL> assert ( len ( dset ) == nPos + nNeg ) <EOL> v = float ( nPos ) / ( nPos + nNeg ) <EOL> mvAcc = max ( [ v , <NUM_LIT:1> - v ] ) <EOL> logger . info ( <EOL> '<STR_LIT>' , <EOL> nPos , <EOL> nNeg , <EOL> mvAcc ) <EOL> statDic = { '<STR_LIT>' : nPos , '<STR_LIT>' : nNeg , '<STR_LIT>' : mvAcc } <EOL> cvResDic = { } <EOL> bContinue = False <EOL> didCV = False <EOL> if ( doCV ) : <EOL> try : <EOL> cvResDic = dssgRunCV ( self . _binaryClassifier , dset ) <EOL> strList = ( <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' ) <EOL> msg = '<STR_LIT>' % ( cvResDic [ strList [ <NUM_LIT:0> ] ] , strList [ <NUM_LIT:1> ] , cvResDic [ strList [ <NUM_LIT:1> ] ] , strList [ <EOL> <NUM_LIT:2> ] , cvResDic [ strList [ <NUM_LIT:2> ] ] [ <NUM_LIT:0> ] , cvResDic [ strList [ <NUM_LIT:2> ] ] [ <NUM_LIT:1> ] ) <EOL> for i in [ <NUM_LIT:1> , <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:5> ] : <EOL> msg += '<STR_LIT>' % ( <EOL> strList [ i ] , <EOL> cvResDic [ strList [ i ] ] ) <EOL> if ( i != <NUM_LIT:5> ) : <EOL> msg += '<STR_LIT:\n>' <EOL> didCV = True <EOL> except TooSkewedLabelsException : <EOL> msg = '<STR_LIT>' <EOL> logger . info ( '<STR_LIT:U+0020>' + msg ) <EOL> statDic [ '<STR_LIT>' ] = cvResDic <EOL> trainStats [ cat ] = statDic <EOL> if ( isinstance ( binaryClassifier , DssgBinaryClassifierAdaptiveInterpolation ) or <EOL> isinstance ( binaryClassifier , DssgBinaryClassifierAdaptiveSVC ) ) : <EOL> binaryClassifier . train ( dset ) <EOL> else : <EOL> if ( nPos == <NUM_LIT:0> or nNeg == <NUM_LIT:0> ) : <EOL> mvClassifier = DssgBinaryClassifierMajorityVote ( ) <EOL> mvClassifier . train ( dset ) <EOL> binaryClassifier = mvClassifier <EOL> else : <EOL> binaryClassifier . train ( dset ) <EOL> if ( isinstance ( binaryClassifier , DssgBinaryClassifierAdaptiveInterpolation ) or isinstance ( binaryClassifier , DssgBinaryClassifierAdaptiveSVC ) or isinstance ( binaryClassifier , DssgBinaryClassifierMajorityVote ) ) : <EOL> logger . info ( <EOL> '<STR_LIT>' , <EOL> repr ( binaryClassifier ) ) <EOL> else : <EOL> nVoca = len ( binaryClassifier . _vectorizer . getFeatureNameList ( ) ) <EOL> logger . info ( '<STR_LIT>' , nVoca ) <EOL> if ( isinstance ( binaryClassifier . _vectorizer , DssgMultiVectorizer ) ) : <EOL> logger . info ( <EOL> '<STR_LIT>' , <EOL> repr ( binaryClassifier . _vectorizer ) ) <EOL> classifierDic [ cat ] = binaryClassifier <EOL> logger . info ( '<STR_LIT>' , str ( type ( binaryClassifier ) ) ) <EOL> if ( didCV ) : <EOL> totNTP += statDic [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> totNFP += statDic [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> totNTN += statDic [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> totNFN += statDic [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> if ( doCV ) : <EOL> didCVCategoryList = [ ] <EOL> for cat , v in trainStats . iteritems ( ) : <EOL> if ( len ( v [ '<STR_LIT>' ] ) != <NUM_LIT:0> ) : <EOL> didCVCategoryList . append ( cat ) <EOL> logger . info ( '<STR_LIT>' , str ( didCVCategoryList ) ) <EOL> meanF1 = np . mean ( <EOL> np . array ( map ( lambda cat : trainStats [ cat ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] , didCVCategoryList ) ) ) <EOL> logger . info ( '<STR_LIT>' , meanF1 ) <EOL> totPrecision = float ( totNTP ) / ( totNTP + totNFP ) <EOL> totRecall = float ( totNTP ) / ( totNTP + totNFN ) <EOL> logger . info ( <EOL> '<STR_LIT>' , <EOL> totPrecision , totRecall , sp . stats . hmean ( [ totPrecision , totRecall ] ) ) <EOL> self . _classifierDic = classifierDic <EOL> self . _categoryList = categoryList <EOL> self . _trainStats = trainStats <EOL> pass <EOL> def predictScore ( self , message ) : <EOL> """<STR_LIT>""" <EOL> labelProbDic = { } <EOL> for cat in self . _categoryList : <EOL> probs = self . _classifierDic [ cat ] . predictScore ( message ) <EOL> labelProbDic [ cat ] = probs [ '<STR_LIT>' ] <EOL> return labelProbDic <EOL> def predictProba ( self , message ) : <EOL> """<STR_LIT>""" <EOL> labelProbDic = { } <EOL> for cat in self . _categoryList : <EOL> probs = self . _classifierDic [ cat ] . predictProba ( message ) <EOL> labelProbDic [ cat ] = probs [ '<STR_LIT>' ] <EOL> return labelProbDic <EOL> pass <EOL> def dssgRunCV ( binaryClassifierObj , <EOL> dsetBinary , <EOL> nFolds = <NUM_LIT:5> ) : <EOL> """<STR_LIT>""" <EOL> y = map ( lambda x : <NUM_LIT:1> if x [ <NUM_LIT:1> ] == '<STR_LIT>' else <NUM_LIT:0> , dsetBinary ) <EOL> ySum = sum ( y ) <EOL> if ySum < <NUM_LIT:2> or len ( y ) - ySum < <NUM_LIT:2> : <EOL> raise TooSkewedLabelsException ( ) <EOL> folds = StratifiedKFold ( y , nFolds ) <EOL> accList = [ ] <EOL> precList = [ ] <EOL> recallList = [ ] <EOL> f1List = [ ] <EOL> allTruey = [ x [ <NUM_LIT:1> ] for x in dsetBinary ] <EOL> allPredy = [ float ( '<STR_LIT>' ) ] * len ( dsetBinary ) <EOL> for trainIdx , testIdx in folds : <EOL> train = [ dsetBinary [ i ] for i in trainIdx ] <EOL> test = [ dsetBinary [ i ] for i in testIdx ] <EOL> binaryClassifier = copy . deepcopy ( binaryClassifierObj ) <EOL> binaryClassifier . train ( train ) <EOL> predy = [ ] <EOL> for x in test : <EOL> predScore = binaryClassifier . predictScore ( x [ <NUM_LIT:0> ] ) <EOL> predy . append ( '<STR_LIT>' if predScore [ '<STR_LIT>' ] >= <NUM_LIT:0> else '<STR_LIT>' ) <EOL> truey = [ x [ <NUM_LIT:1> ] for x in test ] <EOL> acc = metrics . accuracy_score ( truey , predy ) <EOL> accList . append ( acc ) <EOL> for i in range ( len ( test ) ) : <EOL> allPredy [ testIdx [ i ] ] = predy [ i ] <EOL> prec = metrics . precision_score ( allTruey , allPredy , pos_label = '<STR_LIT>' ) <EOL> recall = metrics . recall_score ( allTruey , allPredy , pos_label = '<STR_LIT>' ) <EOL> f1 = metrics . f1_score ( allTruey , allPredy , pos_label = '<STR_LIT>' ) <EOL> nTP = sum ( <EOL> [ allTruey [ i ] == '<STR_LIT>' and allPredy [ i ] == '<STR_LIT>' for i in range ( len ( allTruey ) ) ] ) <EOL> nFP = sum ( <EOL> [ allTruey [ i ] == '<STR_LIT>' and allPredy [ i ] == '<STR_LIT>' for i in range ( len ( allTruey ) ) ] ) <EOL> nTN = sum ( <EOL> [ allTruey [ i ] == '<STR_LIT>' and allPredy [ i ] == '<STR_LIT>' for i in range ( len ( allTruey ) ) ] ) <EOL> nFN = sum ( <EOL> [ allTruey [ i ] == '<STR_LIT>' and allPredy [ i ] == '<STR_LIT>' for i in range ( len ( allTruey ) ) ] ) <EOL> resDic = { } <EOL> resDic [ '<STR_LIT>' ] = nFolds <EOL> me = np . mean ( accList ) <EOL> resDic [ '<STR_LIT>' ] = me <EOL> stderr = stats . sem ( accList ) <EOL> resDic [ '<STR_LIT>' ] = stderr <EOL> alpha = <NUM_LIT> <EOL> dev = stderr * stats . t . ppf ( <NUM_LIT:1> - alpha / <NUM_LIT:2> , len ( accList ) - <NUM_LIT:1> ) <EOL> resDic [ '<STR_LIT>' ] = ( me - dev , me + dev ) <EOL> resDic [ '<STR_LIT>' ] = prec <EOL> resDic [ '<STR_LIT>' ] = recall <EOL> resDic [ '<STR_LIT>' ] = f1 <EOL> resDic [ '<STR_LIT>' ] = nTP <EOL> resDic [ '<STR_LIT>' ] = nFP <EOL> resDic [ '<STR_LIT>' ] = nTN <EOL> resDic [ '<STR_LIT>' ] = nFN <EOL> return resDic </s>
