<s> """<STR_LIT>""" <EOL> def simple_init ( interface , params ) : <EOL> return params <EOL> def map_init ( interface , params ) : <EOL> """<STR_LIT>""" <EOL> import random <EOL> import numpy as np <EOL> random . seed ( params [ '<STR_LIT>' ] ) <EOL> np . random . seed ( params [ '<STR_LIT>' ] ) <EOL> return params <EOL> def map_fit ( interface , state , label , inp ) : <EOL> import numpy as np <EOL> import decision_tree , measures <EOL> from collections import Counter <EOL> out = interface . output ( <NUM_LIT:0> ) <EOL> num_samples = sum ( [ <NUM_LIT:1> for row in inp if len ( row . strip ( ) . split ( state [ "<STR_LIT>" ] ) ) > <NUM_LIT:1> ] ) <EOL> missing_vals_attr = set ( ) <EOL> for counter in range ( state [ "<STR_LIT>" ] ) : <EOL> bag_indices = Counter ( np . random . randint ( num_samples , size = ( num_samples ) ) ) <EOL> attr_mapping , y_mapping = { } , { } <EOL> x , y , fill_in_values = [ ] , [ ] , [ ] <EOL> row_num = <NUM_LIT:0> <EOL> for row in inp : <EOL> row = row . strip ( ) . split ( state [ "<STR_LIT>" ] ) <EOL> if len ( row ) > <NUM_LIT:1> : <EOL> while bag_indices [ row_num ] > <NUM_LIT:0> : <EOL> new_row = [ ] <EOL> for i , j in enumerate ( state [ "<STR_LIT>" ] ) : <EOL> if row [ j ] in state [ "<STR_LIT>" ] : <EOL> new_row . append ( row [ j ] ) <EOL> missing_vals_attr . add ( i ) <EOL> elif state [ "<STR_LIT>" ] [ i ] == "<STR_LIT:c>" : <EOL> new_row . append ( row [ j ] ) <EOL> else : <EOL> if row [ j ] not in attr_mapping : <EOL> attr_mapping [ row [ j ] ] = len ( attr_mapping ) <EOL> new_row . append ( attr_mapping [ row [ j ] ] ) <EOL> x . append ( new_row ) <EOL> if row [ state [ "<STR_LIT>" ] ] not in y_mapping : <EOL> y_mapping [ row [ state [ "<STR_LIT>" ] ] ] = len ( y_mapping ) <EOL> y . append ( y_mapping [ row [ state [ "<STR_LIT>" ] ] ] ) <EOL> bag_indices [ row_num ] -= <NUM_LIT:1> <EOL> row_num += <NUM_LIT:1> <EOL> attr_mapping = { v : k for k , v in attr_mapping . iteritems ( ) } <EOL> y_mapping = { v : k for k , v in y_mapping . iteritems ( ) } <EOL> if len ( y_mapping ) == <NUM_LIT:1> : <EOL> print "<STR_LIT>" <EOL> return <EOL> if len ( missing_vals_attr ) > <NUM_LIT:0> : <EOL> for i in range ( len ( state [ "<STR_LIT>" ] ) ) : <EOL> if state [ "<STR_LIT>" ] [ i ] == "<STR_LIT:c>" : <EOL> value = np . average ( [ sample [ i ] for sample in x if type ( sample [ i ] ) == float ] ) <EOL> fill_in_values . append ( value ) <EOL> else : <EOL> value = np . bincount ( [ sample [ i ] for sample in x if type ( sample [ i ] ) == int ] ) . argmax ( ) <EOL> fill_in_values . append ( attr_mapping [ value ] ) <EOL> if i in missing_vals_attr : <EOL> for j in range ( len ( x ) ) : <EOL> if x [ j ] [ i ] in state [ "<STR_LIT>" ] : <EOL> x [ j ] [ i ] = value <EOL> x = np . array ( x , dtype = np . float32 ) <EOL> y = np . array ( y , dtype = np . uint16 ) <EOL> tree = decision_tree . fit ( <EOL> x = x , <EOL> y = y , <EOL> t = state [ "<STR_LIT>" ] , <EOL> randomized = True , <EOL> max_tree_nodes = state [ "<STR_LIT>" ] , <EOL> min_samples_leaf = state [ "<STR_LIT>" ] , <EOL> min_samples_split = state [ "<STR_LIT>" ] , <EOL> class_majority = state [ "<STR_LIT>" ] , <EOL> measure = measures . info_gain if state [ "<STR_LIT>" ] == "<STR_LIT>" else measures . mdl , <EOL> accuracy = state [ "<STR_LIT>" ] , <EOL> separate_max = state [ "<STR_LIT>" ] ) <EOL> if len ( tree ) < <NUM_LIT:2> : <EOL> continue <EOL> print "<STR_LIT>" <EOL> tree_mapped = { } <EOL> for k , v in tree . iteritems ( ) : <EOL> tree_mapped [ k ] = [ None for i in range ( <NUM_LIT:2> ) ] <EOL> for i , node in enumerate ( v ) : <EOL> dist_map = dict ( [ ( y_mapping [ label ] , freq ) for label , freq in node [ <NUM_LIT:3> ] . iteritems ( ) ] ) <EOL> split_map = set ( [ attr_mapping [ int ( s ) ] for s in list ( node [ <NUM_LIT:2> ] ) ] ) if node [ <NUM_LIT:5> ] == "<STR_LIT:d>" else node [ <NUM_LIT:2> ] <EOL> tree_mapped [ k ] [ i ] = ( node [ <NUM_LIT:0> ] , node [ <NUM_LIT:1> ] , split_map , dist_map , node [ <NUM_LIT:4> ] , node [ <NUM_LIT:5> ] ) <EOL> out . add ( "<STR_LIT>" , tree_mapped ) <EOL> out . add ( "<STR_LIT>" , fill_in_values ) <EOL> def reduce_fit ( interface , state , label , inp ) : <EOL> import numpy as np <EOL> out = interface . output ( <NUM_LIT:0> ) <EOL> out . add ( "<STR_LIT>" , state [ "<STR_LIT>" ] ) <EOL> forest = [ ] <EOL> group_fillins = [ ] <EOL> for i , ( k , value ) in enumerate ( inp ) : <EOL> if k == "<STR_LIT>" : <EOL> forest . append ( value ) <EOL> elif len ( value ) > <NUM_LIT:0> : <EOL> group_fillins . append ( value ) <EOL> out . add ( "<STR_LIT>" , forest ) <EOL> fill_in_values = [ ] <EOL> if len ( group_fillins ) > <NUM_LIT:0> : <EOL> for i , type in enumerate ( state [ "<STR_LIT>" ] ) : <EOL> if type == "<STR_LIT:c>" : <EOL> fill_in_values . append ( np . average ( [ sample [ i ] for sample in group_fillins ] ) ) <EOL> else : <EOL> fill_in_values . append ( np . bincount ( [ sample [ i ] for sample in group_fillins ] ) . argmax ( ) ) <EOL> out . add ( "<STR_LIT>" , fill_in_values ) <EOL> def map_predict_voting ( interface , state , label , inp ) : <EOL> import decision_tree <EOL> out = interface . output ( <NUM_LIT:0> ) <EOL> fill_in_values = state [ "<STR_LIT>" ] <EOL> for row in inp : <EOL> row = row . strip ( ) . split ( state [ "<STR_LIT>" ] ) <EOL> predicted = False <EOL> if len ( row ) > <NUM_LIT:1> : <EOL> x_id = "<STR_LIT>" if state [ "<STR_LIT>" ] == - <NUM_LIT:1> else row [ state [ "<STR_LIT>" ] ] <EOL> x = [ ( fill_in_values [ j ] if row [ j ] in state [ "<STR_LIT>" ] else float ( row [ j ] ) if state [ "<STR_LIT>" ] [ <EOL> i ] == "<STR_LIT:c>" else row [ j ] ) <EOL> for i , j in enumerate ( state [ "<STR_LIT>" ] ) ] <EOL> tallies = { } <EOL> for tree in state [ "<STR_LIT>" ] : <EOL> pred = decision_tree . predict ( tree , x ) <EOL> tallies [ pred ] = tallies . get ( pred , <NUM_LIT:0> ) + <NUM_LIT:1> <EOL> if any ( e > int ( len ( state [ "<STR_LIT>" ] ) / <NUM_LIT> ) for e in tallies . values ( ) ) : <EOL> prediction = max ( tallies , key = tallies . get ) <EOL> out . add ( x_id , ( prediction , tallies [ prediction ] ) ) <EOL> predicted = True <EOL> break <EOL> if not predicted : <EOL> prediction = max ( tallies , key = tallies . get ) <EOL> out . add ( x_id , ( prediction , tallies [ prediction ] ) ) <EOL> def map_predict_dist ( interface , state , label , inp ) : <EOL> import numpy as np <EOL> import decision_tree <EOL> out = interface . output ( <NUM_LIT:0> ) <EOL> ensemble_size = len ( state [ "<STR_LIT>" ] ) <EOL> fill_in_values = state [ "<STR_LIT>" ] <EOL> for row in inp : <EOL> row = row . strip ( ) . split ( state [ "<STR_LIT>" ] ) <EOL> if len ( row ) > <NUM_LIT:1> : <EOL> x_id = "<STR_LIT>" if state [ "<STR_LIT>" ] == - <NUM_LIT:1> else row [ state [ "<STR_LIT>" ] ] <EOL> x = [ ( fill_in_values [ j ] if row [ j ] in state [ "<STR_LIT>" ] else float ( row [ j ] ) if state [ "<STR_LIT>" ] [ <EOL> i ] == "<STR_LIT:c>" else row [ j ] ) <EOL> for i , j in enumerate ( state [ "<STR_LIT>" ] ) ] <EOL> pred_dist = [ decision_tree . predict ( tree , x , dist = True ) for tree in state [ "<STR_LIT>" ] ] <EOL> y_dist = { k : v / float ( ensemble_size ) for k , v in np . sum ( pred_dist ) . iteritems ( ) } <EOL> prediction = max ( y_dist , key = y_dist . get ) <EOL> out . add ( x_id , ( prediction , y_dist [ prediction ] ) ) <EOL> def fit ( dataset , trees_per_chunk = <NUM_LIT:3> , max_tree_nodes = <NUM_LIT:50> , min_samples_leaf = <NUM_LIT:10> , min_samples_split = <NUM_LIT:5> , class_majority = <NUM_LIT:1> , <EOL> measure = "<STR_LIT>" , accuracy = <NUM_LIT:1> , separate_max = True , random_state = None , save_results = True , show = False ) : <EOL> from disco . worker . pipeline . worker import Worker , Stage <EOL> from disco . core import Job <EOL> import discomll <EOL> path = "<STR_LIT:/>" . join ( discomll . __file__ . split ( "<STR_LIT:/>" ) [ : - <NUM_LIT:1> ] + [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> job = Job ( worker = Worker ( save_results = save_results ) ) <EOL> job . pipeline = [ <EOL> ( "<STR_LIT>" , Stage ( "<STR_LIT>" , input_chain = dataset . params [ "<STR_LIT>" ] , init = map_init , process = map_fit ) ) , <EOL> ( '<STR_LIT>' , Stage ( "<STR_LIT>" , init = simple_init , process = reduce_fit , combine = True ) ) ] <EOL> try : <EOL> trees_per_chunk = int ( trees_per_chunk ) <EOL> max_tree_nodes = int ( max_tree_nodes ) if max_tree_nodes != None else max_tree_nodes <EOL> min_samples_leaf = int ( min_samples_leaf ) <EOL> min_samples_split = int ( min_samples_split ) <EOL> class_majority = float ( class_majority ) <EOL> accuracy = int ( accuracy ) <EOL> if trees_per_chunk <= <NUM_LIT:0> or min_samples_leaf <= <NUM_LIT:0> or class_majority <= <NUM_LIT:0> or min_samples_split <= <NUM_LIT:0> and accuracy < <NUM_LIT:0> : <EOL> raise Exception ( "<STR_LIT>" ) <EOL> except ValueError : <EOL> raise Exception ( "<STR_LIT>" ) <EOL> if measure not in [ "<STR_LIT>" , "<STR_LIT>" ] : <EOL> raise Exception ( "<STR_LIT>" ) <EOL> job . params = dataset . params <EOL> job . params [ "<STR_LIT>" ] = trees_per_chunk <EOL> job . params [ "<STR_LIT>" ] = max_tree_nodes <EOL> job . params [ "<STR_LIT>" ] = min_samples_leaf <EOL> job . params [ "<STR_LIT>" ] = min_samples_split <EOL> job . params [ "<STR_LIT>" ] = class_majority <EOL> job . params [ "<STR_LIT>" ] = measure <EOL> job . params [ "<STR_LIT>" ] = accuracy <EOL> job . params [ "<STR_LIT>" ] = separate_max <EOL> job . params [ '<STR_LIT>' ] = random_state <EOL> job . run ( name = "<STR_LIT>" , input = dataset . params [ "<STR_LIT>" ] , <EOL> required_files = [ path + "<STR_LIT>" , path + "<STR_LIT>" ] ) <EOL> fitmodel_url = job . wait ( show = show ) <EOL> return { "<STR_LIT>" : fitmodel_url } <EOL> def predict ( dataset , fitmodel_url , voting = False , save_results = True , show = False ) : <EOL> from disco . worker . pipeline . worker import Worker , Stage <EOL> from disco . core import Job , result_iterator <EOL> import discomll <EOL> path = "<STR_LIT:/>" . join ( discomll . __file__ . split ( "<STR_LIT:/>" ) [ : - <NUM_LIT:1> ] + [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> if "<STR_LIT>" not in fitmodel_url : <EOL> raise Exception ( "<STR_LIT>" ) <EOL> job = Job ( worker = Worker ( save_results = save_results ) ) <EOL> job . pipeline = [ ( "<STR_LIT>" , Stage ( "<STR_LIT>" , input_chain = dataset . params [ "<STR_LIT>" ] , init = simple_init , <EOL> process = map_predict_voting if voting else map_predict_dist ) ) ] <EOL> job . params = dataset . params <EOL> for k , v in result_iterator ( fitmodel_url [ "<STR_LIT>" ] ) : <EOL> job . params [ k ] = v <EOL> if len ( job . params [ "<STR_LIT>" ] ) == <NUM_LIT:0> : <EOL> print "<STR_LIT>" <EOL> return [ ] <EOL> job . run ( name = "<STR_LIT>" , input = dataset . params [ "<STR_LIT>" ] , <EOL> required_files = [ path + "<STR_LIT>" ] ) <EOL> return job . wait ( show = show ) </s>
