def validateClassifier(self, cut_detect, cut_train, cut_geometry, y_pred):
        """
        Make some diagnostic plots
        """

        color = {'detect': 'Red',
                 'nondetect': 'Gold',
                 'why_not': 'none',
                 'actual': 'DodgerBlue',
                 'hsc': 'lime'}
        size = {'detect': 5,
                'nondetect': 5,
                'why_not': 35,
                'actual': None,
                'hsc': None}
        marker = {'detect': 'o',
                  'nondetect': 'o',
                  'why_not': 'o',
                  'actual': 's',
                  'hsc': 's'}
        alpha = {'detect': None,
                 'nondetect': None,
                 'why_not': None,
                 'actual': None,
                 'hsc': None}
        edgecolor = {'detect': None,
                     'nondetect': None,
                     'why_not': 'magenta',
                     'actual': 'black',
                     'hsc': 'black'}

        title = 'N_train = %i ; N_test = %i'%(len(cut_train),len(cut_test))        import matplotlib
        cmap = matplotlib.colors.ListedColormap(['Gold', 'Orange', 'DarkOrange', 'OrangeRed', 'Red'])

        pylab.figure()
        pylab.xscale('log')
        pylab.scatter(1.e3 * self.data_population['r_physical'][cut_train & cut_geometry], 
                      self.data_population['abs_mag'][cut_train & cut_geometry], 
                      c=cut_detect[cut_train & cut_geometry].astype(int), vmin=0., vmax=1., s=size['detect'], cmap=cmap, label=None) 
        pylab.scatter(1.e3 * self.data_population['r_physical'][~cut_train  & cut_geometry], 
                      self.data_population['abs_mag'][~cut_train & cut_geometry], 
                      c=y_pred, edgecolor='black', vmin=0., vmax=1., s=(3 * size['detect']), cmap=cmap, label=None) 
        colorbar = pylab.colorbar()
        colorbar.set_label('ML Predicted Detection Probability')
        pylab.scatter(0., 0., s=(3 * size['detect']), c='none', edgecolor='black', label='Test')
        #pylab.scatter(1.e3 * r_physical_actual, abs_mag_actual, 
        #              c=color['actual'], s=size['actual'], marker=marker['actual'], edgecolor=edgecolor['actual'], alpha=alpha['actual'], label='Actual MW Satellites')
        #pylab.scatter(1.e3 * r_physical_actual[cut_hsc], abs_mag_actual[cut_hsc], 
        #              c=color['hsc'], s=size['hsc'], marker=marker['hsc'], edgecolor=edgecolor['hsc'], alpha=alpha['hsc'], label='Actual MW Satellites: HSC')
        pylab.xlim(1., 3.e3)
        pylab.ylim(6., -12.)
        pylab.xlabel('Half-light Radius (pc)')
        pylab.ylabel('M_V (mag)')
        pylab.legend(loc='upper left', markerscale=2)
        pylab.title(title)

        import ugali.utils.bayesian_efficiency # Replace with standalone util

        bins = np.linspace(0., 1., 10 + 1)
        centers = np.empty(len(bins) - 1)
        bin_prob = np.empty(len(bins) - 1)
        bin_prob_err_hi = np.empty(len(bins) - 1)
        bin_prob_err_lo = np.empty(len(bins) - 1)
        bin_counts = np.empty(len(bins) - 1)
        for ii in range(0, len(centers)):
            cut_bin = (y_pred > bins[ii]) & (y_pred < bins[ii + 1])
            centers[ii] = np.mean(y_pred[cut_bin])
    
            n_trials = np.sum(cut_bin)
            n_successes = np.sum(cut_detect[~cut_train & cut_geometry] & cut_bin)
            efficiency, errorbar = ugali.utils.bayesian_efficiency.bayesianInterval(n_trials, n_successes, errorbar=True)
            bin_prob[ii] = efficiency
            bin_prob_err_hi[ii] = errorbar[1]
            bin_prob_err_lo[ii] = errorbar[0]
            #bin_prob[ii] = float(np.sum(cut_detect[~cut_train] & cut_bin)) / np.sum(cut_bin)
            #bin_prob[ii] = np.mean(y[cut].astype(float))
            bin_counts[ii] = np.sum(cut_bin)
    
        pylab.figure()
        pylab.plot([0., 1.], [0., 1.], c='black', ls='--')
        pylab.errorbar(centers, bin_prob, yerr=[bin_prob_err_lo, bin_prob_err_hi], c='red')
        pylab.plot(centers, bin_prob, c='red')
        pylab.scatter(centers, bin_prob, c=bin_counts, edgecolor='red', s=50, cmap='Reds', zorder=999)
        colorbar = pylab.colorbar()
        colorbar.set_label('Counts')
        pylab.xlabel('ML Predicted Detection Probability')
        pylab.ylabel('Fraction Detected')
        pylab.xlim(0., 1.)
        pylab.ylim(0., 1.)
        pylab.title(title)
def validateClassifier ( self , cut_detect , cut_train , cut_geometry , y_pred ) : color = { 'detect' : 'Red' , 'nondetect' : 'Gold' , 'why_not' : 'none' , 'actual' : 'DodgerBlue' , 'hsc' : 'lime' } size = { 'detect' : 5 , 'nondetect' : 5 , 'why_not' : 35 , 'actual' : None , 'hsc' : None } marker = { 'detect' : 'o' , 'nondetect' : 'o' , 'why_not' : 'o' , 'actual' : 's' , 'hsc' : 's' } alpha = { 'detect' : None , 'nondetect' : None , 'why_not' : None , 'actual' : None , 'hsc' : None } edgecolor = { 'detect' : None , 'nondetect' : None , 'why_not' : 'magenta' , 'actual' : 'black' , 'hsc' : 'black' } title = 'N_train = %i ; N_test = %i' % ( len ( cut_train ) , len ( cut_test ) ) import matplotlib cmap = matplotlib . colors . ListedColormap ( [ 'Gold' , 'Orange' , 'DarkOrange' , 'OrangeRed' , 'Red' ] ) pylab . figure ( ) pylab . xscale ( 'log' ) pylab . scatter ( 1.e3 * self . data_population [ 'r_physical' ] [ cut_train & cut_geometry ] , self . data_population [ 'abs_mag' ] [ cut_train & cut_geometry ] , c = cut_detect [ cut_train & cut_geometry ] . astype ( int ) , vmin = 0. , vmax = 1. , s = size [ 'detect' ] , cmap = cmap , label = None ) pylab . scatter ( 1.e3 * self . data_population [ 'r_physical' ] [ ~ cut_train & cut_geometry ] , self . data_population [ 'abs_mag' ] [ ~ cut_train & cut_geometry ] , c = y_pred , edgecolor = 'black' , vmin = 0. , vmax = 1. , s = ( 3 * size [ 'detect' ] ) , cmap = cmap , label = None ) colorbar = pylab . colorbar ( ) colorbar . set_label ( 'ML Predicted Detection Probability' ) pylab . scatter ( 0. , 0. , s = ( 3 * size [ 'detect' ] ) , c = 'none' , edgecolor = 'black' , label = 'Test' ) pylab . xlim ( 1. , 3.e3 ) pylab . ylim ( 6. , - 12. ) pylab . xlabel ( 'Half-light Radius (pc)' ) pylab . ylabel ( 'M_V (mag)' ) pylab . legend ( loc = 'upper left' , markerscale = 2 ) pylab . title ( title ) import ugali . utils . bayesian_efficiency bins = np . linspace ( 0. , 1. , 10 + 1 ) centers = np . empty ( len ( bins ) - 1 ) bin_prob = np . empty ( len ( bins ) - 1 ) bin_prob_err_hi = np . empty ( len ( bins ) - 1 ) bin_prob_err_lo = np . empty ( len ( bins ) - 1 ) bin_counts = np . empty ( len ( bins ) - 1 ) for ii in range ( 0 , len ( centers ) ) : cut_bin = ( y_pred > bins [ ii ] ) & ( y_pred < bins [ ii + 1 ] ) centers [ ii ] = np . mean ( y_pred [ cut_bin ] ) n_trials = np . sum ( cut_bin ) n_successes = np . sum ( cut_detect [ ~ cut_train & cut_geometry ] & cut_bin ) efficiency , errorbar = ugali . utils . bayesian_efficiency . bayesianInterval ( n_trials , n_successes , errorbar = True ) bin_prob [ ii ] = efficiency bin_prob_err_hi [ ii ] = errorbar [ 1 ] bin_prob_err_lo [ ii ] = errorbar [ 0 ] bin_counts [ ii ] = np . sum ( cut_bin ) pylab . figure ( ) pylab . plot ( [ 0. , 1. ] , [ 0. , 1. ] , c = 'black' , ls = '--' ) pylab . errorbar ( centers , bin_prob , yerr = [ bin_prob_err_lo , bin_prob_err_hi ] , c = 'red' ) pylab . plot ( centers , bin_prob , c = 'red' ) pylab . scatter ( centers , bin_prob , c = bin_counts , edgecolor = 'red' , s = 50 , cmap = 'Reds' , zorder = 999 ) colorbar = pylab . colorbar ( ) colorbar . set_label ( 'Counts' ) pylab . xlabel ( 'ML Predicted Detection Probability' ) pylab . ylabel ( 'Fraction Detected' ) pylab . xlim ( 0. , 1. ) pylab . ylim ( 0. , 1. ) pylab . title ( title )
Make some diagnostic plots