<s> import sys <EOL> import copy <EOL> sys . path . append ( '<STR_LIT:..>' ) <EOL> import time <EOL> import numpy as np <EOL> import scipy as sp <EOL> import scipy . stats <EOL> import sklearn <EOL> import xgboost <EOL> import xgboost . sklearn <EOL> import explainers <EOL> from load_datasets import * <EOL> import glob <EOL> import argparse <EOL> import collections <EOL> from sklearn import ensemble , cross_validation <EOL> import pickle <EOL> import parzen_windows <EOL> def submodular_fn ( explanations , feature_value ) : <EOL> """<STR_LIT>""" <EOL> z_words = set ( ) <EOL> for exp in explanations : <EOL> z_words = z_words . union ( [ x [ <NUM_LIT:0> ] for x in exp ] ) <EOL> normalizer = sum ( [ feature_value [ w ] for w in z_words ] ) <EOL> def fnz ( x ) : <EOL> all_words = set ( ) <EOL> for doc in x : <EOL> all_words = all_words . union ( [ x [ <NUM_LIT:0> ] for x in explanations [ doc ] ] ) <EOL> return sum ( [ feature_value [ w ] for w in all_words ] ) / normalizer <EOL> fnz . num_items = len ( explanations ) <EOL> return fnz <EOL> def greedy ( submodular_fn , k , chosen = [ ] ) : <EOL> chosen = copy . deepcopy ( chosen ) <EOL> all_items = range ( submodular_fn . num_items ) <EOL> current_value = <NUM_LIT:0> <EOL> while len ( chosen ) != k : <EOL> best_gain = <NUM_LIT:0> <EOL> best_item = all_items [ <NUM_LIT:0> ] <EOL> for i in all_items : <EOL> gain = submodular_fn ( chosen + [ i ] ) - current_value <EOL> if gain > best_gain : <EOL> best_gain = gain <EOL> best_item = i <EOL> chosen . append ( best_item ) <EOL> all_items . remove ( best_item ) <EOL> current_value += best_gain <EOL> return chosen <EOL> def submodular_pick ( pickled_map , explainer , B , use_explanation_weights = False , <EOL> alternate = False ) : <EOL> def get_function ( exps ) : <EOL> feature_value = collections . defaultdict ( float ) <EOL> for exp in exps : <EOL> for f , v in exp : <EOL> if not use_explanation_weights : <EOL> v = <NUM_LIT:1> <EOL> feature_value [ f ] += np . abs ( v ) <EOL> for f in feature_value : <EOL> feature_value [ f ] = np . sqrt ( feature_value [ f ] ) <EOL> submodular = submodular_fn ( exps , feature_value ) <EOL> return submodular <EOL> out = greedy ( submodular , B ) <EOL> return out <EOL> if explainer in [ '<STR_LIT>' , '<STR_LIT>' ] : <EOL> exps1 = [ x [ <NUM_LIT:0> ] for x in pickled_map [ '<STR_LIT>' ] [ explainer ] ] <EOL> exps2 = [ x [ <NUM_LIT:0> ] for x in pickled_map [ '<STR_LIT>' ] [ explainer ] ] <EOL> else : <EOL> exps1 = pickled_map [ '<STR_LIT>' ] [ explainer ] <EOL> exps2 = pickled_map [ '<STR_LIT>' ] [ explainer ] <EOL> fn1 = get_function ( exps1 ) <EOL> fn2 = get_function ( exps2 ) <EOL> if not alternate : <EOL> return greedy ( fn1 , B ) , greedy ( fn2 , B ) <EOL> else : <EOL> ret = [ ] <EOL> for i in range ( B ) : <EOL> fn = fn1 if i % <NUM_LIT:2> == <NUM_LIT:0> else fn2 <EOL> ret = greedy ( fn , i + <NUM_LIT:1> , ret ) <EOL> return ret <EOL> def all_pick ( pickled_map , explainer , B ) : <EOL> list_ = range ( len ( pickled_map [ '<STR_LIT>' ] [ explainer ] ) ) <EOL> return list_ , list_ <EOL> def random_pick ( pickled_map , explainer , B ) : <EOL> list_ = np . random . choice ( range ( len ( pickled_map [ '<STR_LIT>' ] [ explainer ] ) ) , B , replace = False ) <EOL> return list_ , list_ <EOL> def find_untrustworthy ( explainer , exps , instances , untrustworthy ) : <EOL> found = set ( ) <EOL> for i in instances : <EOL> if explainer in [ '<STR_LIT>' , '<STR_LIT>' ] : <EOL> exp , mean = exps [ i ] <EOL> else : <EOL> exp = exps [ i ] <EOL> found = found . union ( [ x [ <NUM_LIT:0> ] for x in exp if x [ <NUM_LIT:0> ] in untrustworthy ] ) <EOL> return found <EOL> def tally_mistrust ( explainer , exps , predict_probas , untrustworthy ) : <EOL> trust_fn = lambda prev , curr : ( prev > <NUM_LIT:0.5> and curr > <NUM_LIT:0.5> ) or ( prev <= <NUM_LIT:0.5> and curr <= <NUM_LIT:0.5> ) <EOL> trust_fn_all = lambda exp , unt : len ( [ x [ <NUM_LIT:0> ] for x in exp if x [ <NUM_LIT:0> ] in unt ] ) == <NUM_LIT:0> <EOL> mistrust = <NUM_LIT:0> <EOL> for i in range ( len ( exps ) ) : <EOL> if explainer in [ '<STR_LIT>' , '<STR_LIT>' ] : <EOL> exp , mean = exps [ i ] <EOL> if explainer == '<STR_LIT>' : <EOL> prev_tot = sum ( [ x [ <NUM_LIT:1> ] for x in exp ] ) + mean <EOL> elif explainer == '<STR_LIT>' : <EOL> prev_tot = mean <EOL> tot = prev_tot - sum ( [ x [ <NUM_LIT:1> ] for x in exp if x [ <NUM_LIT:0> ] in untrustworthy ] ) <EOL> if not trust_fn ( tot , prev_tot ) : <EOL> mistrust += <NUM_LIT:1> <EOL> else : <EOL> exp = exps [ i ] <EOL> if not trust_fn_all ( exp , untrustworthy ) : <EOL> mistrust += <NUM_LIT:1> <EOL> return mistrust <EOL> def main ( ) : <EOL> parser = argparse . ArgumentParser ( description = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , '<STR_LIT>' , type = str , required = True , help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , '<STR_LIT>' , type = str , required = True , help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , '<STR_LIT>' , type = int , required = True , help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , '<STR_LIT>' , type = str , required = False , default = '<STR_LIT:all>' , help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , '<STR_LIT>' , type = int , required = False , default = <NUM_LIT:1> , help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , '<STR_LIT>' , type = int , required = False , default = <NUM_LIT:10> , help = '<STR_LIT>' ) <EOL> args = parser . parse_args ( ) <EOL> dataset = args . dataset <EOL> got_right = lambda test1 , test2 , mistrust1 , mistrust2 : mistrust1 < mistrust2 if test1 > test2 else mistrust1 > mistrust2 <EOL> names = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> num_exps = <NUM_LIT:0> <EOL> B = args . num_instances <EOL> rounds = <NUM_LIT:1> <EOL> if args . pick == '<STR_LIT:all>' : <EOL> pick_function = all_pick <EOL> elif args . pick == '<STR_LIT>' : <EOL> pick_function = lambda a , b , c : submodular_pick ( a , b , c , use_explanation_weights = True ) <EOL> elif args . pick == '<STR_LIT>' : <EOL> pick_function = random_pick <EOL> rounds = args . num_rounds <EOL> accuracy = collections . defaultdict ( lambda : [ ] ) <EOL> right = collections . defaultdict ( lambda : [ ] ) <EOL> for r in range ( rounds ) : <EOL> right = collections . defaultdict ( lambda : [ ] ) <EOL> for filez in glob . glob ( os . path . join ( args . output_folder , '<STR_LIT>' % args . dataset ) ) [ : <NUM_LIT> ] : <EOL> num_exps += <NUM_LIT:1> <EOL> pickled_map = pickle . load ( open ( filez ) ) <EOL> predict_probas = pickled_map [ '<STR_LIT>' ] <EOL> predict_probas2 = pickled_map [ '<STR_LIT>' ] <EOL> test1 = pickled_map [ '<STR_LIT>' ] <EOL> test2 = pickled_map [ '<STR_LIT>' ] <EOL> untrustworthy = pickled_map [ '<STR_LIT>' ] <EOL> for explainer in names : <EOL> if explainer . startswith ( '<STR_LIT>' ) : <EOL> pick1 , pick2 = pick_function ( pickled_map , '<STR_LIT>' , B ) <EOL> exps1 = pickled_map [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> exps2 = pickled_map [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> elif explainer . startswith ( '<STR_LIT>' ) : <EOL> pick1 , pick2 = pick_function ( pickled_map , '<STR_LIT>' , B ) <EOL> exps1 = pickled_map [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> exps2 = pickled_map [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> else : <EOL> pick1 , pick2 = pick_function ( pickled_map , explainer , B ) <EOL> exps1 = pickled_map [ '<STR_LIT>' ] [ explainer ] <EOL> exps2 = pickled_map [ '<STR_LIT>' ] [ explainer ] <EOL> if args . pick != '<STR_LIT:all>' : <EOL> unt1 = find_untrustworthy ( explainer , exps1 , pick1 , untrustworthy ) <EOL> unt2 = find_untrustworthy ( explainer , exps2 , pick2 , untrustworthy ) <EOL> else : <EOL> unt1 = unt2 = untrustworthy <EOL> mistrust1 = tally_mistrust ( explainer , exps1 , predict_probas , unt1 ) <EOL> mistrust2 = tally_mistrust ( explainer , exps2 , predict_probas2 , unt2 ) <EOL> while mistrust1 == mistrust2 : <EOL> mistrust1 = np . random . randint ( <NUM_LIT:0> , <NUM_LIT:10> ) <EOL> mistrust2 = np . random . randint ( <NUM_LIT:0> , <NUM_LIT:10> ) <EOL> right [ explainer ] . append ( int ( got_right ( test1 , test2 , mistrust1 , mistrust2 ) ) ) <EOL> right [ '<STR_LIT>' ] . append ( int ( got_right ( test1 , test2 , np . random . random ( ) , np . random . random ( ) ) ) ) <EOL> for name in right : <EOL> accuracy [ name ] . append ( np . mean ( right [ name ] ) ) <EOL> print '<STR_LIT>' <EOL> for name in right : <EOL> print name , np . mean ( accuracy [ name ] ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> main ( ) </s>
