<s> import Orange <EOL> import datetime <EOL> def writeConfusionToFile ( classifications , data , name ) : <EOL> with open ( name , '<STR_LIT:w>' ) as writeTo : <EOL> writeTo . write ( '<STR_LIT>' + '<STR_LIT:U+002C>' . join ( data . domain . class_var . values ) + '<STR_LIT:\n>' ) <EOL> for label in data . domain . class_var . values : <EOL> writeTo . write ( label + '<STR_LIT:U+002C>' + '<STR_LIT:U+002C>' . join ( [ str ( classifications [ ( str ( label2 ) , str ( label ) ) ] ) for label2 in data . domain . class_var . values ] ) + '<STR_LIT:\n>' ) <EOL> def testSuccess ( classifier , test , classifications = None ) : <EOL> if classifications is None : <EOL> classifications = { } <EOL> for label in test . domain . class_var . values : <EOL> for label2 in test . domain . class_var . values : <EOL> if ( str ( label ) , str ( label2 ) ) not in classifications : <EOL> classifications [ ( str ( label ) , str ( label2 ) ) ] = <NUM_LIT:0> <EOL> correct = <NUM_LIT:0> <EOL> wrong = <NUM_LIT:0> <EOL> for example in test : <EOL> try : <EOL> val = classifier ( example ) <EOL> except : <EOL> return '<STR_LIT>' <EOL> classifications [ ( str ( val ) , str ( example [ '<STR_LIT:label>' ] ) ) ] += <NUM_LIT:1.0> <EOL> if val == example [ '<STR_LIT:label>' ] : <EOL> correct += <NUM_LIT:1.0> <EOL> else : <EOL> wrong += <NUM_LIT:1.0> <EOL> return ( correct , wrong , classifications ) <EOL> def getPercentSuccess ( result ) : <EOL> return result [ <NUM_LIT:0> ] / ( result [ <NUM_LIT:0> ] + result [ <NUM_LIT:1> ] ) <EOL> def runTest ( train , test , learner , totalClassifications = None , writeConfusion = False ) : <EOL> start = datetime . datetime . today ( ) <EOL> classifier = learner ( train ) <EOL> end = datetime . datetime . today ( ) <EOL> print '<STR_LIT>' % ( str ( start ) , str ( end ) , str ( end - start ) ) <EOL> resultTrain = testSuccess ( classifier , train , totalClassifications ) <EOL> resultTest = testSuccess ( classifier , test , totalClassifications ) <EOL> if writeConfusion : <EOL> writeConfusionToFile ( resultTest [ <NUM_LIT:2> ] , test , '<STR_LIT>' % learner . name ) <EOL> return ( resultTrain , resultTest , classifier ) <EOL> def kFoldCrossValidation ( data , learner , returnBest = False , folds = <NUM_LIT:10> ) : <EOL> indices = Orange . data . sample . SubsetIndicesCV ( data , folds ) <EOL> totalSuccessTrain = <NUM_LIT:0> <EOL> totalSuccessTest = <NUM_LIT:0> <EOL> best = [ <NUM_LIT:0> , None ] <EOL> totalClassifications = { } <EOL> for label in data . domain . class_var . values : <EOL> for label2 in data . domain . class_var . values : <EOL> totalClassifications [ ( str ( label ) , str ( label2 ) ) ] = <NUM_LIT:0> <EOL> for fold in range ( folds ) : <EOL> train = data . select_ref ( indices , fold , negate = <NUM_LIT:1> ) <EOL> test = data . select_ref ( indices , fold ) <EOL> print "<STR_LIT>" % fold <EOL> resultTrain , resultTest , classifier = runTest ( train , test , learner , totalClassifications ) <EOL> if resultTest == '<STR_LIT>' : <EOL> print '<STR_LIT>' <EOL> folds = folds - <NUM_LIT:1> <EOL> else : <EOL> ( correct , wrong , classifications ) = resultTrain <EOL> successTrain = correct / ( correct + wrong ) <EOL> ( correct , wrong , classifications ) = resultTest <EOL> successTest = correct / ( correct + wrong ) <EOL> if best [ <NUM_LIT:0> ] < successTest : <EOL> best [ <NUM_LIT:1> ] = classifier <EOL> best [ <NUM_LIT:0> ] = successTest <EOL> totalSuccessTest += successTest <EOL> totalSuccessTrain += successTrain <EOL> print '<STR_LIT>' % ( successTrain , successTest ) <EOL> writeConfusionToFile ( classifications , test , '<STR_LIT>' % learner . name ) <EOL> averageSuccessTest = totalSuccessTest / folds <EOL> averageSuccessTrain = totalSuccessTrain / folds <EOL> print '<STR_LIT>' % ( averageSuccessTrain , averageSuccessTest ) <EOL> print '<STR_LIT>' <EOL> if returnBest : <EOL> return ( best [ <NUM_LIT:1> ] , averageSuccess ) <EOL> return ( averageSuccessTrain , averageSuccessTest ) <EOL> def testTreeLearning ( data , test = None , getBest = False , folds = <NUM_LIT:10> , writeConfusion = False , ** kw ) : <EOL> print '<STR_LIT>' <EOL> tree = Orange . classification . tree . TreeLearner ( name = '<STR_LIT>' , ** kw ) <EOL> if test is not None : <EOL> ( resultTrain , resultTest , classifier ) = runTest ( data , test , tree , writeConfusion = writeConfusion ) <EOL> return ( getPercentSuccess ( resultTrain ) , getPercentSuccess ( resultTest ) ) <EOL> return kFoldCrossValidation ( data , tree , returnBest = getBest , folds = folds ) <EOL> def testNeuralNetLearning ( data , test = None , getBest = False , folds = <NUM_LIT:10> , writeConfusion = False , ** kw ) : <EOL> print '<STR_LIT>' <EOL> net = Orange . classification . neural . NeuralNetworkLearner ( name = "<STR_LIT>" , ** kw ) <EOL> if test is not None : <EOL> ( resultTrain , resultTest , classifier ) = runTest ( data , test , net , writeConfusion = writeConfusion ) <EOL> return ( getPercentSuccess ( resultTrain ) , getPercentSuccess ( resultTest ) ) <EOL> return kFoldCrossValidation ( data , net , returnBest = getBest , folds = folds ) <EOL> def testKNNLearning ( data , test = None , getBest = False , folds = <NUM_LIT:10> , k = <NUM_LIT:10> , writeConfusion = False ) : <EOL> print '<STR_LIT>' <EOL> knnLearner = Orange . classification . knn . kNNLearner ( name = '<STR_LIT>' , k = k ) <EOL> if test is not None : <EOL> ( resultTrain , resultTest , classifier ) = runTest ( data , test , knnLearner , writeConfusion = writeConfusion ) <EOL> return ( getPercentSuccess ( resultTrain ) , getPercentSuccess ( resultTest ) ) <EOL> return kFoldCrossValidation ( data , knnLearner , returnBest = getBest , folds = folds ) <EOL> def testSVMLearning ( data , test = None , getBest = False , folds = <NUM_LIT:10> , writeConfusion = False , ** kw ) : <EOL> print '<STR_LIT>' <EOL> svmLearner = Orange . classification . svm . SVMLearner ( name = '<STR_LIT>' , ** kw ) <EOL> if test is not None : <EOL> ( resultTrain , resultTest , classifier ) = runTest ( data , test , svmLearner , writeConfusion = writeConfusion ) <EOL> return ( getPercentSuccess ( resultTrain ) , getPercentSuccess ( resultTest ) ) <EOL> return kFoldCrossValidation ( data , svmLearner , returnBest = getBest , folds = folds ) <EOL> def testBoostLearning ( data , learner , test = None , getBest = False , folds = <NUM_LIT:10> , t = <NUM_LIT:10> , writeConfusion = False ) : <EOL> print '<STR_LIT>' <EOL> boostLearner = Orange . ensemble . boosting . BoostedLearner ( learner , name = '<STR_LIT>' , t = t ) <EOL> if test is not None : <EOL> ( resultTrain , resultTest , classifier ) = runTest ( data , test , boostLearner , writeConfusion = writeConfusion ) <EOL> return ( getPercentSuccess ( resultTrain ) , getPercentSuccess ( resultTest ) ) <EOL> return kFoldCrossValidation ( data , boostLearner , returnBest = getBest , folds = folds ) <EOL> def writeToCVS ( fileName , values , mode = '<STR_LIT:w>' ) : <EOL> with open ( fileName , mode ) as writeTo : <EOL> for valList in values : <EOL> writeTo . write ( '<STR_LIT:U+002C>' . join ( [ str ( v ) for v in valList ] ) + '<STR_LIT:\n>' ) <EOL> def getDataWrapper ( maxes , fileName , methodCall ) : <EOL> successesTrain = [ '<STR_LIT>' ] <EOL> successesTest = [ '<STR_LIT>' ] <EOL> for m in maxes [ <NUM_LIT:1> : ] : <EOL> success = methodCall ( m ) <EOL> successesTrain . append ( success [ <NUM_LIT:0> ] ) <EOL> successesTest . append ( success [ <NUM_LIT:1> ] ) <EOL> writeToCVS ( dataName + fileName + '<STR_LIT>' , [ maxes , successesTrain , successesTest ] ) <EOL> def getTreeData ( data , testData = None , fileName = '<STR_LIT>' , pruneDepth = False ) : <EOL> if pruneDepth : <EOL> maxes = [ '<STR_LIT>' ] + [ i * <NUM_LIT:5> for i in range ( <NUM_LIT:1> , <NUM_LIT:5> ) ] + [ i * <NUM_LIT:10> for i in range ( <NUM_LIT:3> , <NUM_LIT:11> ) ] <EOL> getDataWrapper ( maxes , fileName , lambda m : testTreeLearning ( data , test = testData , max_depth = m , same_majority_pruning = True ) ) <EOL> else : <EOL> maxes = [ '<STR_LIT>' ] + [ i * <NUM_LIT> for i in range ( <NUM_LIT:4> , <NUM_LIT> ) ] <EOL> getDataWrapper ( maxes , fileName , lambda m : testTreeLearning ( data , teste = testData , max_majority = m , same_majority_pruning = True ) ) <EOL> def getKNNData ( data , testData = None , fileName = '<STR_LIT>' ) : <EOL> maxes = [ '<STR_LIT:k>' ] + [ i * <NUM_LIT:5> for i in range ( <NUM_LIT:1> , <NUM_LIT> , <NUM_LIT:2> ) ] <EOL> getDataWrapper ( maxes , fileName , lambda m : testKNNLearning ( data , test = testData , k = m ) ) <EOL> def getSVMData ( data , testData = None , kernel = Orange . classification . svm . kernels . RBF , fileName = '<STR_LIT>' ) : <EOL> maxes = [ '<STR_LIT>' ] + [ <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT:0.1> ] <EOL> getDataWrapper ( maxes , fileName , lambda m : testSVMLearning ( data , test = testData , gamma = m , kernel_type = kernel ) ) <EOL> def getNNData ( data , testData = None , fileName = '<STR_LIT>' , getEpochs = False ) : <EOL> if getEpochs : <EOL> maxes = [ '<STR_LIT>' ] + [ i * <NUM_LIT:200> for i in range ( <NUM_LIT:1> , <NUM_LIT> ) ] <EOL> getDataWrapper ( maxes , fileName , lambda m : testNeuralNetLearning ( data , test = testData , max_iter = m ) ) <EOL> else : <EOL> maxes = [ '<STR_LIT>' ] + [ <NUM_LIT:5> , <NUM_LIT:10> , <NUM_LIT> , <NUM_LIT:50> , <NUM_LIT:100> , <NUM_LIT> , <NUM_LIT:1000> , <NUM_LIT> , <NUM_LIT> ] <EOL> getDataWrapper ( maxes , fileName , lambda m : testNeuralNetLearning ( data , test = testData , n_mid = m ) ) <EOL> def getBoostingData ( data , testData = None , fileName = '<STR_LIT>' ) : <EOL> maxes = [ '<STR_LIT:t>' ] + [ i * <NUM_LIT:4> for i in range ( <NUM_LIT:1> , <NUM_LIT:6> ) ] <EOL> tree = Orange . classification . tree . TreeLearner ( name = '<STR_LIT>' , max_depth = <NUM_LIT> , same_majority_pruning = True ) <EOL> getDataWrapper ( maxes , fileName , lambda m : testBoostLearning ( data , tree , test = testData , t = m ) ) <EOL> def getLearningCurves ( ) : <EOL> tree = Orange . classification . tree . TreeLearner ( name = '<STR_LIT>' , same_majority_pruning = True ) <EOL> successes = [ [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ] <EOL> for i in range ( <NUM_LIT:1> , <NUM_LIT:2> ) : <EOL> newNumInstances = int ( ( i / <NUM_LIT:1> ) * numInstances ) - <NUM_LIT:200> <EOL> success = [ newNumInstances ] <EOL> dataSubset = data . get_items_ref ( range ( newNumInstances ) ) <EOL> success . append ( testTreeLearning ( dataSubset , same_majority_pruning = True ) ) <EOL> success . append ( testNeuralNetLearning ( dataSubset , n_mid = <NUM_LIT> , normalize = False ) ) <EOL> success . append ( testKNNLearning ( dataSubset ) ) <EOL> success . append ( testSVMLearning ( dataSubset , gamma = <NUM_LIT> ) ) <EOL> success . append ( testBoostLearning ( dataSubset , tree , t = <NUM_LIT:4> ) ) <EOL> successes . append ( success ) <EOL> writeToCVS ( '<STR_LIT>' , successes ) <EOL> dataName = '<STR_LIT>' <EOL> data = Orange . data . Table ( "<STR_LIT>" ) <EOL> numInstances = len ( data ) <EOL> data . shuffle ( ) <EOL> getNNData ( data ) </s>
