original_docstring: 
          Performs a complete evaluation of the model predictions.
  
          Input : preds, type, foldername
              - preds:        a list where each element 0<=preds[i]<=1 gives the probability of chromatin accessibility, in the
                              same order as that of the corresponding eval_generator
              - type:         'validation' or 'test'
              - foldername:   folder where all analysis files will be stored
              - report_filename:   name of text file where consolidated report is stored
          
docstring: Performs a complete evaluation of the model predictions.
docstring_tokens: Performs a complete evaluation of the model predictions .
code: def evaluate_model(self, preds, type, foldername, report_filename='report'):
          """
          Performs a complete evaluation of the model predictions.
  
          Input : preds, type, foldername
              - preds:        a list where each element 0<=preds[i]<=1 gives the probability of chromatin accessibility, in the
                              same order as that of the corresponding eval_generator
              - type:         'validation' or 'test'
              - foldername:   folder where all analysis files will be stored
              - report_filename:   name of text file where consolidated report is stored
          """
  
          _, avail_cell_types, avail_chrms = self._map_type(type)
          
          f = open(foldername + '/' + report_filename + '.txt', 'w')
          f.write('-'*17 + '\n' + 'EVALUATION REPORT\n' + '-'*17 + '\n')
          f.write('TYPE: ' + type + '\n')
          f.write('HELD OUT : ' + self.hold_out + '\n')
          f.write(type + ' chromosomes : ' + str([avail_chrms]) + '\n')
          f.write(type + ' cell types  : ' + str([avail_cell_types]) + '\n\n')
  
          preds = np.array(preds)
          
          # ctype_chr_pred_dict[ctype][chrm] will have keys 'preds' and 'labels'
          ctype_chr_pred_dict, labels = self.populate_ctype_chr_pred_dict(avail_cell_types, avail_chrms, preds, ret_labels=True)      
          assert(labels.size==preds.size)
  
          os.system('mkdir ' + foldername + '/plots')
  
          # overall stats
          auprc, auc, prec, rec = self._auprc_auc_pr(labels, preds)
          f.write('OVERALL AUPRC = {0:0.03f}, AUC = {1:0.03f}'.format(auprc,auc) + '\n')
  
          if self.hold_out == 'cell_types':
              ctype_auprcs, ctype_aucs, ctype_precs, ctype_recs, ctype_legends = [], [], [], [], []
              self._update_stats(ctype_auprcs, ctype_aucs, ctype_precs, ctype_recs, ctype_legends, auprc, auc, prec, rec, 'OVERALL AUPRC = {0:0.03f} , AUC = {1:0.03f}'.format(auprc,auc))
  
              for ctype in avail_cell_types:
                  # ctype stats
                  auprcs, aucs, precs, recs, legends = [], [], [], [], []
  
                  auprc, auc, prec, rec = self._auprc_auc_pr(np.hstack([ctype_chr_pred_dict[ctype][x]['labels'] for x in ctype_chr_pred_dict[ctype]]), np.hstack([ctype_chr_pred_dict[ctype][x]['preds'] for x in ctype_chr_pred_dict[ctype]]))
                  self._update_stats(auprcs, aucs, precs, recs, legends, auprc, auc, prec, rec, 'CELL TYPE : ' + str(ctype) + ' AUPRC = {0:0.03f} , AUC = {1:0.03f}'.format(auprc,auc))
                  self._update_stats(ctype_auprcs, ctype_aucs, ctype_precs, ctype_recs, ctype_legends, auprc, auc, prec, rec, 'CELL TYPE : ' + str(ctype) + ' AUPRC = {0:0.03f} , AUC = {1:0.03f}'.format(auprc,auc))
                  f.write('\nCELL TYPE : ' + str(ctype) + ' - AUPRC = {0:0.03f} , AUC = {1:0.03f}'.format(auprc,auc) + '\n')
  
                  for chrm in avail_chrms:
                      # ctype x chrm stats
                      auprc, auc, prec, rec = self._auprc_auc_pr(ctype_chr_pred_dict[ctype][chrm]['labels'], ctype_chr_pred_dict[ctype][chrm]['preds'])
                      self._update_stats(auprcs, aucs, precs, recs, legends, auprc, auc, prec, rec, 'CHRM ' + str(chrm) + ', AUPRC : {0:0.03f} , AUC : {1:0.03f}'.format(auprc,auc))
                      f.write('CELL TYPE : ' + str(ctype) + ' CHRM : ' + str(chrm) + ' - AUPRC = {0:0.03f} , AUC = {1:0.03f}'.format(auprc,auc) + '\n')
  
                  self._save_plot(precs, recs, legends, foldername + '/plots', 'ctype_' + str(ctype), 'CELL TYPE : ' + str(ctype))
              self._save_plot(ctype_precs, ctype_recs, ctype_legends, foldername + '/plots', 'summary', 'SUMMARY')
  
          elif self.hold_out == 'chromosomes':
              chrm_auprcs, chrm_aucs, chrm_precs, chrm_recs, chrm_legends = [], [], [], [], []
              self._update_stats(chrm_auprcs,chrm_aucs, chrm_precs,chrm_recs, chrm_legends, auprc, auc, prec, rec, 'OVERALL AUPRC = {0:0.03f} , AUC = {1:0.03f}'.format(auprc, auc))
  
              for chrm in avail_chrms:
                  #chrm stats
                  auprcs, aucs, precs, recs, legends = [], [], [], [], []
  
                  auprc, auc, prec, rec = self._auprc_auc_pr(np.hstack([ctype_chr_pred_dict[x][chrm]['labels'] for x in ctype_chr_pred_dict]), np.hstack([ctype_chr_pred_dict[x][chrm]['preds'] for x in ctype_chr_pred_dict]))
                  self._update_stats(auprcs, aucs, precs, recs, legends, auprc, auc, prec, rec, 'CHRM : ' + str(chrm) + ' AUPRC = {0:0.03f} , AUC = {1:0.03f}'.format(auprc,auc))
                  self._update_stats(chrm_auprcs,chrm_aucs,chrm_precs,chrm_recs, chrm_legends, auprc, auc, prec, rec, 'CHRM : ' + str(chrm) + ' AUPRC = {0:0.03f} , AUC = {1:0.03f}'.format(auprc,auc))
                  f.write('\nCHRM : ' + str(chrm) + ' - AUPRC = {0:0.03f} , AUC = {1:0.03f}'.format(auprc,auc) + '\n')
  
                  for ctype in avail_cell_types:
                      # ctype x chrm stats
                      auprc, auc, prec, rec = self._auprc_auc_pr(ctype_chr_pred_dict[ctype][chrm]['labels'], ctype_chr_pred_dict[ctype][chrm]['preds'])
                      self._update_stats(auprcs, aucs, precs, recs, legends, auprc, auc, prec, rec, 'CELL TYPE ' + str(ctype) + ', AUPRC : {0:0.03f} , AUC : {1:0.03f}'.format(auprc,auc))
                      f.write('CELL TYPE : ' + str(ctype) + ' CHRM : ' + str(chrm) + ' - AUPRC = {0:0.03f} , AUC = {1:0.03f}'.format(auprc,auc) + '\n')
  
                  self._save_plot(precs, recs, legends, foldername + '/plots', 'chrm_' + str(chrm), 'CHRM : ' + str(chrm))
              self._save_plot(chrm_precs, chrm_recs, chrm_legends, foldername + '/plots', 'summary', 'SUMMARY')
  
          f.close()
code_tokens: def evaluate_model ( self , preds , type , foldername , report_filename = 'report' ) : _ , avail_cell_types , avail_chrms = self . _map_type ( type ) f = open ( foldername + '/' + report_filename + '.txt' , 'w' ) f . write ( '-' * 17 + '\n' + 'EVALUATION REPORT\n' + '-' * 17 + '\n' ) f . write ( 'TYPE: ' + type + '\n' ) f . write ( 'HELD OUT : ' + self . hold_out + '\n' ) f . write ( type + ' chromosomes : ' + str ( [ avail_chrms ] ) + '\n' ) f . write ( type + ' cell types  : ' + str ( [ avail_cell_types ] ) + '\n\n' ) preds = np . array ( preds ) ctype_chr_pred_dict , labels = self . populate_ctype_chr_pred_dict ( avail_cell_types , avail_chrms , preds , ret_labels = True ) assert ( labels . size == preds . size ) os . system ( 'mkdir ' + foldername + '/plots' ) auprc , auc , prec , rec = self . _auprc_auc_pr ( labels , preds ) f . write ( 'OVERALL AUPRC = {0:0.03f}, AUC = {1:0.03f}' . format ( auprc , auc ) + '\n' ) if self . hold_out == 'cell_types' : ctype_auprcs , ctype_aucs , ctype_precs , ctype_recs , ctype_legends = [ ] , [ ] , [ ] , [ ] , [ ] self . _update_stats ( ctype_auprcs , ctype_aucs , ctype_precs , ctype_recs , ctype_legends , auprc , auc , prec , rec , 'OVERALL AUPRC = {0:0.03f} , AUC = {1:0.03f}' . format ( auprc , auc ) ) for ctype in avail_cell_types : auprcs , aucs , precs , recs , legends = [ ] , [ ] , [ ] , [ ] , [ ] auprc , auc , prec , rec = self . _auprc_auc_pr ( np . hstack ( [ ctype_chr_pred_dict [ ctype ] [ x ] [ 'labels' ] for x in ctype_chr_pred_dict [ ctype ] ] ) , np . hstack ( [ ctype_chr_pred_dict [ ctype ] [ x ] [ 'preds' ] for x in ctype_chr_pred_dict [ ctype ] ] ) ) self . _update_stats ( auprcs , aucs , precs , recs , legends , auprc , auc , prec , rec , 'CELL TYPE : ' + str ( ctype ) + ' AUPRC = {0:0.03f} , AUC = {1:0.03f}' . format ( auprc , auc ) ) self . _update_stats ( ctype_auprcs , ctype_aucs , ctype_precs , ctype_recs , ctype_legends , auprc , auc , prec , rec , 'CELL TYPE : ' + str ( ctype ) + ' AUPRC = {0:0.03f} , AUC = {1:0.03f}' . format ( auprc , auc ) ) f . write ( '\nCELL TYPE : ' + str ( ctype ) + ' - AUPRC = {0:0.03f} , AUC = {1:0.03f}' . format ( auprc , auc ) + '\n' ) for chrm in avail_chrms : auprc , auc , prec , rec = self . _auprc_auc_pr ( ctype_chr_pred_dict [ ctype ] [ chrm ] [ 'labels' ] , ctype_chr_pred_dict [ ctype ] [ chrm ] [ 'preds' ] ) self . _update_stats ( auprcs , aucs , precs , recs , legends , auprc , auc , prec , rec , 'CHRM ' + str ( chrm ) + ', AUPRC : {0:0.03f} , AUC : {1:0.03f}' . format ( auprc , auc ) ) f . write ( 'CELL TYPE : ' + str ( ctype ) + ' CHRM : ' + str ( chrm ) + ' - AUPRC = {0:0.03f} , AUC = {1:0.03f}' . format ( auprc , auc ) + '\n' ) self . _save_plot ( precs , recs , legends , foldername + '/plots' , 'ctype_' + str ( ctype ) , 'CELL TYPE : ' + str ( ctype ) ) self . _save_plot ( ctype_precs , ctype_recs , ctype_legends , foldername + '/plots' , 'summary' , 'SUMMARY' ) elif self . hold_out == 'chromosomes' : chrm_auprcs , chrm_aucs , chrm_precs , chrm_recs , chrm_legends = [ ] , [ ] , [ ] , [ ] , [ ] self . _update_stats ( chrm_auprcs , chrm_aucs , chrm_precs , chrm_recs , chrm_legends , auprc , auc , prec , rec , 'OVERALL AUPRC = {0:0.03f} , AUC = {1:0.03f}' . format ( auprc , auc ) ) for chrm in avail_chrms : auprcs , aucs , precs , recs , legends = [ ] , [ ] , [ ] , [ ] , [ ] auprc , auc , prec , rec = self . _auprc_auc_pr ( np . hstack ( [ ctype_chr_pred_dict [ x ] [ chrm ] [ 'labels' ] for x in ctype_chr_pred_dict ] ) , np . hstack ( [ ctype_chr_pred_dict [ x ] [ chrm ] [ 'preds' ] for x in ctype_chr_pred_dict ] ) ) self . _update_stats ( auprcs , aucs , precs , recs , legends , auprc , auc , prec , rec , 'CHRM : ' + str ( chrm ) + ' AUPRC = {0:0.03f} , AUC = {1:0.03f}' . format ( auprc , auc ) ) self . _update_stats ( chrm_auprcs , chrm_aucs , chrm_precs , chrm_recs , chrm_legends , auprc , auc , prec , rec , 'CHRM : ' + str ( chrm ) + ' AUPRC = {0:0.03f} , AUC = {1:0.03f}' . format ( auprc , auc ) ) f . write ( '\nCHRM : ' + str ( chrm ) + ' - AUPRC = {0:0.03f} , AUC = {1:0.03f}' . format ( auprc , auc ) + '\n' ) for ctype in avail_cell_types : auprc , auc , prec , rec = self . _auprc_auc_pr ( ctype_chr_pred_dict [ ctype ] [ chrm ] [ 'labels' ] , ctype_chr_pred_dict [ ctype ] [ chrm ] [ 'preds' ] ) self . _update_stats ( auprcs , aucs , precs , recs , legends , auprc , auc , prec , rec , 'CELL TYPE ' + str ( ctype ) + ', AUPRC : {0:0.03f} , AUC : {1:0.03f}' . format ( auprc , auc ) ) f . write ( 'CELL TYPE : ' + str ( ctype ) + ' CHRM : ' + str ( chrm ) + ' - AUPRC = {0:0.03f} , AUC = {1:0.03f}' . format ( auprc , auc ) + '\n' ) self . _save_plot ( precs , recs , legends , foldername + '/plots' , 'chrm_' + str ( chrm ) , 'CHRM : ' + str ( chrm ) ) self . _save_plot ( chrm_precs , chrm_recs , chrm_legends , foldername + '/plots' , 'summary' , 'SUMMARY' ) f . close ( )
short_docstring: Performs a complete evaluation of the model predictions.
short_docstring_tokens: Performs a complete evaluation of the model predictions .
comment: """
          Performs a complete evaluation of the model predictions.
  
          Input : preds, type, foldername
              - preds:        a list where each element 0<=preds[i]<=1 gives the probability of chromatin accessibility, in the
                              same order as that of the corresponding eval_generator
              - type:         'validation' or 'test'
              - foldername:   folder where all analysis files will be stored
              - report_filename:   name of text file where consolidated report is stored
          """$SEP$# ctype_chr_pred_dict[ctype][chrm] will have keys 'preds' and 'labels'$SEP$# overall stats$SEP$# ctype stats$SEP$# ctype x chrm stats$SEP$#chrm stats$SEP$# ctype x chrm stats