<s> from . . models import Experiment , Component <EOL> from toposort import toposort_flatten <EOL> from . . ml_models import Classifier <EOL> from rest_framework import viewsets <EOL> from django . http import HttpResponse <EOL> from collections import Counter <EOL> from pandas import * <EOL> import threading <EOL> import json <EOL> CACHE = { } <EOL> class myThread ( threading . Thread ) : <EOL> def __init__ ( self , thread_id , name , experiment , component_id , max_results , cache_results ) : <EOL> threading . Thread . __init__ ( self ) <EOL> self . threadID = thread_id <EOL> self . name = name <EOL> self . experiment = experiment <EOL> self . comp_id = component_id <EOL> self . result = { } <EOL> self . max_results = max_results <EOL> self . cache_results = cache_results <EOL> def run ( self ) : <EOL> print "<STR_LIT>" , self . name , "<STR_LIT>" , self . comp_id <EOL> exp = Experiment . objects . get ( pk = self . experiment ) <EOL> graph = exp . workflow . graph_data <EOL> graph_data = { } <EOL> print graph <EOL> status = "<STR_LIT:success>" <EOL> err_msg = "<STR_LIT>" <EOL> tmp = graph . split ( '<STR_LIT:U+002C>' ) <EOL> for elem in tmp : <EOL> node = elem . split ( "<STR_LIT::>" ) <EOL> if len ( node ) > <NUM_LIT:1> : <EOL> first_node = node [ <NUM_LIT:0> ] <EOL> second_node = node [ <NUM_LIT:1> ] <EOL> else : <EOL> first_node = node [ <NUM_LIT:0> ] <EOL> second_node = '<STR_LIT>' <EOL> if second_node in graph_data : <EOL> depend_nodes = graph_data [ second_node ] <EOL> depend_nodes . add ( first_node ) <EOL> else : <EOL> graph_data [ second_node ] = set ( ) <EOL> graph_data [ second_node ] . add ( first_node ) <EOL> topological_graph = toposort_flatten ( graph_data ) <EOL> print "<STR_LIT>" , topological_graph <EOL> if self . experiment in CACHE : <EOL> input_data = CACHE [ self . experiment ] <EOL> else : <EOL> input_data = DataFrame <EOL> feature_names = None <EOL> feature_types = None <EOL> output_data = None <EOL> for data in topological_graph : <EOL> component_id = int ( data ) <EOL> comp = Component . objects . get ( pk = component_id ) <EOL> print "<STR_LIT>" , component_id , "<STR_LIT:U+0020>" , comp . operation_type <EOL> op = comp . operation_type <EOL> if op . function_type == '<STR_LIT>' : <EOL> if op . function_arg == '<STR_LIT>' : <EOL> if op . function_subtype == '<STR_LIT>' : <EOL> filename = op . function_subtype_arg <EOL> input_data = read_csv ( filename ) <EOL> feature_names = input_data . columns <EOL> if op . function_arg == '<STR_LIT>' : <EOL> if op . function_subtype == '<STR_LIT>' : <EOL> row_values = json . loads ( op . function_subtype_arg ) <EOL> input_data . loc [ len ( input_data ) + <NUM_LIT:1> ] = row_values <EOL> if op . function_arg == '<STR_LIT>' : <EOL> if op . function_subtype == '<STR_LIT>' : <EOL> params = json . loads ( op . function_subtype_arg ) <EOL> train_data_percentage = int ( params [ "<STR_LIT>" ] ) <EOL> target_column = int ( params [ "<STR_LIT>" ] ) <EOL> model_type = op . function_arg_id <EOL> print model_type , train_data_percentage , target_column <EOL> target_feature = feature_names [ target_column ] <EOL> try : <EOL> actual_target_column = input_data . columns . get_loc ( target_feature ) <EOL> input_feature_columns = range ( len ( input_data . columns ) ) <EOL> input_feature_columns . remove ( actual_target_column ) <EOL> input_features = input_data . columns [ input_feature_columns ] <EOL> classifier = Classifier ( <EOL> input_data , model_type , train_data_percentage , <EOL> input_features , target_feature ) <EOL> output_data = classifier . learn ( ) <EOL> except ValueError as e : <EOL> status = "<STR_LIT>" <EOL> err_msg = "<STR_LIT>" <EOL> except KeyError as e : <EOL> status = "<STR_LIT>" <EOL> err_msg = target_feature + "<STR_LIT>" <EOL> if op . function_type == '<STR_LIT>' : <EOL> if op . function_arg == '<STR_LIT>' : <EOL> if op . function_subtype == '<STR_LIT>' : <EOL> feature_types = json . loads ( op . function_subtype_arg ) <EOL> print "<STR_LIT>" , feature_names , "<STR_LIT>" , feature_types <EOL> if op . function_arg == '<STR_LIT>' : <EOL> if op . function_subtype == '<STR_LIT>' : <EOL> constant_value = float ( op . function_subtype_arg ) <EOL> column_id = float ( op . function_arg_id ) <EOL> column_name = feature_names [ column_id ] <EOL> if column_name not in input_data : <EOL> status = "<STR_LIT>" <EOL> err_msg = column_name + "<STR_LIT>" <EOL> elif input_data [ column_name ] . dtype == '<STR_LIT:object>' : <EOL> status = "<STR_LIT>" <EOL> err_msg = "<STR_LIT>" + column_name + "<STR_LIT>" <EOL> else : <EOL> input_data [ column_name ] += constant_value <EOL> if op . function_subtype == '<STR_LIT>' : <EOL> constant_value = float ( op . function_subtype_arg ) <EOL> column_id = float ( op . function_arg_id ) <EOL> column_name = feature_names [ column_id ] <EOL> if column_name not in input_data : <EOL> status = "<STR_LIT>" <EOL> err_msg = column_name + "<STR_LIT>" <EOL> elif input_data [ column_name ] . dtype == '<STR_LIT:object>' : <EOL> status = "<STR_LIT>" <EOL> err_msg = "<STR_LIT>" + column_name + "<STR_LIT>" <EOL> else : <EOL> input_data [ column_name ] -= constant_value <EOL> if op . function_subtype == '<STR_LIT>' : <EOL> constant_value = float ( op . function_subtype_arg ) <EOL> column_id = float ( op . function_arg_id ) <EOL> column_name = feature_names [ column_id ] <EOL> if column_name not in input_data : <EOL> status = "<STR_LIT>" <EOL> err_msg = column_name + "<STR_LIT>" <EOL> elif input_data [ column_name ] . dtype == '<STR_LIT:object>' : <EOL> status = "<STR_LIT>" <EOL> err_msg = "<STR_LIT>" + column_name + "<STR_LIT>" <EOL> else : <EOL> input_data [ column_name ] *= constant_value <EOL> if op . function_subtype == '<STR_LIT>' : <EOL> constant_value = float ( op . function_subtype_arg ) <EOL> column_id = float ( op . function_arg_id ) <EOL> column_name = feature_names [ column_id ] <EOL> if column_name not in input_data : <EOL> status = "<STR_LIT>" <EOL> err_msg = column_name + "<STR_LIT>" <EOL> elif input_data [ column_name ] . dtype == '<STR_LIT:object>' : <EOL> status = "<STR_LIT>" <EOL> err_msg = "<STR_LIT>" + column_name + "<STR_LIT>" <EOL> else : <EOL> input_data [ column_name ] /= constant_value <EOL> if op . function_subtype == '<STR_LIT>' : <EOL> column_id = float ( op . function_arg_id ) <EOL> column_name = feature_names [ column_id ] <EOL> sum_array = input_data . sum ( axis = <NUM_LIT:0> ) <EOL> if column_name not in sum_array : <EOL> status = "<STR_LIT>" <EOL> err_msg = column_name + "<STR_LIT>" <EOL> else : <EOL> normalization_value = sum_array [ column_name ] <EOL> input_data [ column_name ] = input_data [ column_name ] / normalization_value <EOL> if op . function_type == '<STR_LIT>' : <EOL> if op . function_arg == '<STR_LIT>' : <EOL> if op . function_subtype == '<STR_LIT>' : <EOL> column_id_list = json . loads ( op . function_arg_id ) <EOL> excluded_columns = range ( len ( feature_names ) ) <EOL> for elem in column_id_list : <EOL> excluded_columns . remove ( elem ) <EOL> excluded_columns = [ x for x in excluded_columns if feature_names [ x ] in input_data ] <EOL> print "<STR_LIT>" , excluded_columns <EOL> if excluded_columns : <EOL> input_data = input_data . drop ( feature_names [ excluded_columns ] , axis = <NUM_LIT:1> ) <EOL> if op . function_subtype == '<STR_LIT>' : <EOL> column_id_list = json . loads ( op . function_arg_id ) <EOL> column_name_list = [ ] <EOL> for elem in column_id_list : <EOL> column_name = feature_names [ elem ] <EOL> if column_name not in input_data : <EOL> status = "<STR_LIT>" <EOL> err_msg = column_name + "<STR_LIT>" <EOL> else : <EOL> column_name_list . append ( column_name ) <EOL> if column_name_list : <EOL> input_data = input_data . drop_duplicates ( subset = column_name_list ) <EOL> if op . function_subtype == '<STR_LIT>' : <EOL> if op . function_subtype_arg == '<STR_LIT>' : <EOL> input_data = input_data . fillna ( input_data . mean ( ) . round ( <NUM_LIT:2> ) ) <EOL> if op . function_subtype_arg == '<STR_LIT>' : <EOL> input_data = input_data . fillna ( input_data . median ( ) . round ( <NUM_LIT:2> ) ) <EOL> if op . function_subtype_arg == '<STR_LIT>' : <EOL> input_data = input_data . fillna ( input_data . mode ( ) ) <EOL> if op . function_subtype_arg == '<STR_LIT>' : <EOL> input_data = input_data . dropna ( axis = <NUM_LIT:0> ) <EOL> if component_id == self . comp_id : <EOL> print "<STR_LIT>" <EOL> self . result [ "<STR_LIT>" ] = list ( input_data . columns ) <EOL> if feature_types is not None : <EOL> self . result [ "<STR_LIT>" ] = feature_types <EOL> self . result [ "<STR_LIT:data>" ] = [ ] <EOL> result_length = min ( len ( input_data ) , self . max_results ) <EOL> for i in range ( result_length ) : <EOL> tmp = [ ] <EOL> for col in input_data . columns : <EOL> if json . dumps ( input_data [ col ] [ i ] ) == '<STR_LIT>' : <EOL> tmp . append ( '<STR_LIT>' ) <EOL> else : <EOL> tmp . append ( input_data [ col ] [ i ] ) <EOL> self . result [ "<STR_LIT:data>" ] . append ( tmp ) <EOL> self . result [ "<STR_LIT>" ] = [ ] <EOL> for name in list ( input_data . columns ) : <EOL> top_uniques = Counter ( list ( input_data [ name ] ) ) . most_common ( <NUM_LIT:4> ) <EOL> col_names = [ ] <EOL> unique_count = [ ] <EOL> for val in top_uniques : <EOL> if json . dumps ( val [ <NUM_LIT:0> ] ) == '<STR_LIT>' : <EOL> continue <EOL> col_names . append ( val [ <NUM_LIT:0> ] ) <EOL> unique_count . append ( val [ <NUM_LIT:1> ] ) <EOL> tmp = [ col_names , unique_count ] <EOL> self . result [ "<STR_LIT>" ] . append ( tmp ) <EOL> if output_data is not None : <EOL> self . result [ "<STR_LIT>" ] = output_data <EOL> self . result [ "<STR_LIT:status>" ] = status <EOL> self . result [ "<STR_LIT:message>" ] = err_msg <EOL> self . result [ "<STR_LIT>" ] = list ( input_data . isnull ( ) . sum ( ) . values ) <EOL> mean = input_data . mean ( ) . round ( <NUM_LIT:2> ) <EOL> median = input_data . median ( ) . round ( <NUM_LIT:2> ) <EOL> self . result [ "<STR_LIT>" ] = [ ] <EOL> self . result [ "<STR_LIT>" ] = [ ] <EOL> for elem in input_data . columns : <EOL> if elem in mean : <EOL> self . result [ "<STR_LIT>" ] . append ( mean [ elem ] ) <EOL> else : <EOL> self . result [ "<STR_LIT>" ] . append ( '<STR_LIT>' ) <EOL> if elem in median : <EOL> self . result [ "<STR_LIT>" ] . append ( median [ elem ] ) <EOL> else : <EOL> self . result [ "<STR_LIT>" ] . append ( '<STR_LIT>' ) <EOL> self . result [ "<STR_LIT>" ] = [ ] <EOL> for elem in input_data . columns : <EOL> self . result [ "<STR_LIT>" ] . append ( input_data [ elem ] . nunique ( ) ) <EOL> self . result [ "<STR_LIT>" ] = [ ] <EOL> self . result [ "<STR_LIT>" ] = [ ] <EOL> self . result [ "<STR_LIT>" ] = [ ] <EOL> self . result [ "<STR_LIT>" ] = [ ] <EOL> self . result [ "<STR_LIT>" ] = [ ] <EOL> self . result [ "<STR_LIT>" ] = [ ] <EOL> metric_val = input_data . describe ( ) <EOL> for elem in input_data . columns : <EOL> if elem in metric_val : <EOL> val = metric_val [ elem ] . round ( <NUM_LIT:2> ) <EOL> self . result [ "<STR_LIT>" ] . append ( val [ "<STR_LIT>" ] ) <EOL> self . result [ "<STR_LIT>" ] . append ( val [ "<STR_LIT>" ] ) <EOL> self . result [ "<STR_LIT>" ] . append ( val [ "<STR_LIT>" ] ) <EOL> self . result [ "<STR_LIT>" ] . append ( val [ "<STR_LIT>" ] ) <EOL> self . result [ "<STR_LIT>" ] . append ( val [ "<STR_LIT>" ] ) <EOL> self . result [ "<STR_LIT>" ] . append ( val [ "<STR_LIT>" ] ) <EOL> else : <EOL> self . result [ "<STR_LIT>" ] . append ( '<STR_LIT>' ) <EOL> self . result [ "<STR_LIT>" ] . append ( '<STR_LIT>' ) <EOL> self . result [ "<STR_LIT>" ] . append ( '<STR_LIT>' ) <EOL> self . result [ "<STR_LIT>" ] . append ( '<STR_LIT>' ) <EOL> self . result [ "<STR_LIT>" ] . append ( '<STR_LIT>' ) <EOL> self . result [ "<STR_LIT>" ] . append ( '<STR_LIT>' ) <EOL> self . result [ "<STR_LIT>" ] = input_data . shape [ <NUM_LIT:0> ] <EOL> self . result [ "<STR_LIT>" ] = input_data . shape [ <NUM_LIT:1> ] <EOL> if self . cache_results is True : <EOL> CACHE [ self . experiment ] = input_data <EOL> print self . result [ "<STR_LIT:status>" ] <EOL> print self . result [ "<STR_LIT:message>" ] <EOL> break <EOL> class ResultViewSet ( viewsets . ViewSet ) : <EOL> def list ( self , request ) : <EOL> exp_id = int ( request . GET . get ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> component_id = int ( request . GET . get ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> print "<STR_LIT>" , exp_id <EOL> thread = myThread ( <NUM_LIT:1> , "<STR_LIT>" , exp_id , component_id , <NUM_LIT:10> , False ) <EOL> thread . start ( ) <EOL> thread . join ( ) <EOL> return HttpResponse ( json . dumps ( thread . result ) , content_type = "<STR_LIT:application/json>" ) </s>
