def stratify_by_scores(scores, goal_n_strata='auto', method='cum_sqrt_F',
                       n_bins = 'auto'):
    """Stratify by binning the items based on their scores

    Parameters
    ----------
    scores : array-like, shape=(n_items,)
        ordered array of scores which quantify the classifier confidence for
        the items in the pool. High scores indicate a high confidence that
        the true label is a "1" (and vice versa for label "0").

    goal_n_strata : int or 'auto', optional, default 'auto'
        desired number of strata. If set to 'auto', the number is selected
        using the Freedman-Diaconis rule. Note that for the 'cum_sqrt_F' method
        this number is a goal -- the actual number of strata created may be
        less than the goal.

    method : {'cum_sqrt_F' or 'equal_size'}, optional, default 'cum_sqrt_F'
        stratification method to use. 'equal_size' aims to create s

    Other Parameters
    ----------------
    n_bins : int or 'auto', optional, default 'auto'
        specify the number of bins to use when estimating the distribution of
        the score function. This is used when ``goal_n_strata = 'auto'``
        and/or when ``method = 'cum_sqrt_F'``. If set to 'auto', the number is
        selected using the Freedman-Diaconis rule.

    Returns
    -------
    Strata instance
    """

    available_methods = ['equal_size', 'cum_sqrt_F']
    if method not in available_methods:
        raise ValueError("method argument is invalid")

    if (method == 'cum_sqrt_F') or (goal_n_strata == 'auto'):
        # computation below is needed for cum_sqrt_F method OR if we need to
        # determine the number of strata for equal_size method automatically
        if n_bins == 'auto':
            # choose n_bins heuristically
            width_score = _heuristic_bin_width(scores)
            n_bins = np.ceil(sp.ptp(scores)/width_score).astype(int)
            print("Automatically setting n_bins = {}.".format(n_bins))

        # approx distribution of scores -- called F
        counts, score_bins = np.histogram(scores, bins=n_bins)

        # generate cumulative dist of sqrt(F)
        sqrt_counts = np.sqrt(counts)
        csf = np.cumsum(sqrt_counts)

        if goal_n_strata == 'auto':
            # choose heuristically
            width_csf = _heuristic_bin_width(csf)
            goal_n_strata = np.ceil(sp.ptp(csf)/width_csf).astype(int)
            print("Automatically setting goal_n_strata = {}.".format(goal_n_strata))
        elif method == 'cum_sqrt_F':
            width_csf = csf[-1]/goal_n_strata

    # goal_n_strata is now guaranteed to have a valid integer value

    if method == 'equal_size':
        sorted_ids = scores.argsort()
        n_items = len(sorted_ids)
        quotient = n_items // goal_n_strata
        remainder = n_items % goal_n_strata
        allocations = np.empty(n_items, dtype='int')

        st_pops = [quotient for i in range(goal_n_strata)]
        for i in range(remainder):
            st_pops[i] += 1

        j = 0
        for k,nk in enumerate(st_pops):
            start = j
            end = j + nk
            allocations[sorted_ids[start:end]] = k
            j = end

    if method == 'cum_sqrt_F':
        if goal_n_strata > n_bins:
            warnings.warn("goal_n_strata > n_bins. "
                          "Consider increasing n_bins.")
        # calculate roughly equal bins on cum sqrt(F) scale
        csf_bins = [x * width_csf for x in np.arange(goal_n_strata + 1)]

        # map cum sqrt(F) bins to score bins
        j = 0
        new_bins = []
        for (idx,value) in enumerate(csf):
            if j == (len(csf_bins) - 1) or idx == (len(csf) - 1):
                new_bins.append(score_bins[-1])
                break
            if value >= csf_bins[j]:
                new_bins.append(score_bins[idx])
                j += 1
        new_bins[0] -= 0.01
        new_bins[-1] += 0.01

        # bin scores based on new_bins
        allocations = np.digitize(scores, bins=new_bins, right=True) - 1

        # remove empty strata
        nonempty_ids = np.unique(allocations)
        n_strata = len(nonempty_ids)
        indices = np.arange(n_strata)
        allocations = np.digitize(allocations, nonempty_ids, right=True)

        if n_strata < goal_n_strata:
            warnings.warn("Failed to create {} strata".format(goal_n_strata))

    return Strata(allocations)
def stratify_by_scores ( scores , goal_n_strata = 'auto' , method = 'cum_sqrt_F' , n_bins = 'auto' ) : available_methods = [ 'equal_size' , 'cum_sqrt_F' ] if method not in available_methods : raise ValueError ( "method argument is invalid" ) if ( method == 'cum_sqrt_F' ) or ( goal_n_strata == 'auto' ) : if n_bins == 'auto' : width_score = _heuristic_bin_width ( scores ) n_bins = np . ceil ( sp . ptp ( scores ) / width_score ) . astype ( int ) print ( "Automatically setting n_bins = {}." . format ( n_bins ) ) counts , score_bins = np . histogram ( scores , bins = n_bins ) sqrt_counts = np . sqrt ( counts ) csf = np . cumsum ( sqrt_counts ) if goal_n_strata == 'auto' : width_csf = _heuristic_bin_width ( csf ) goal_n_strata = np . ceil ( sp . ptp ( csf ) / width_csf ) . astype ( int ) print ( "Automatically setting goal_n_strata = {}." . format ( goal_n_strata ) ) elif method == 'cum_sqrt_F' : width_csf = csf [ - 1 ] / goal_n_strata if method == 'equal_size' : sorted_ids = scores . argsort ( ) n_items = len ( sorted_ids ) quotient = n_items // goal_n_strata remainder = n_items % goal_n_strata allocations = np . empty ( n_items , dtype = 'int' ) st_pops = [ quotient for i in range ( goal_n_strata ) ] for i in range ( remainder ) : st_pops [ i ] += 1 j = 0 for k , nk in enumerate ( st_pops ) : start = j end = j + nk allocations [ sorted_ids [ start : end ] ] = k j = end if method == 'cum_sqrt_F' : if goal_n_strata > n_bins : warnings . warn ( "goal_n_strata > n_bins. " "Consider increasing n_bins." ) csf_bins = [ x * width_csf for x in np . arange ( goal_n_strata + 1 ) ] j = 0 new_bins = [ ] for ( idx , value ) in enumerate ( csf ) : if j == ( len ( csf_bins ) - 1 ) or idx == ( len ( csf ) - 1 ) : new_bins . append ( score_bins [ - 1 ] ) break if value >= csf_bins [ j ] : new_bins . append ( score_bins [ idx ] ) j += 1 new_bins [ 0 ] -= 0.01 new_bins [ - 1 ] += 0.01 allocations = np . digitize ( scores , bins = new_bins , right = True ) - 1 nonempty_ids = np . unique ( allocations ) n_strata = len ( nonempty_ids ) indices = np . arange ( n_strata ) allocations = np . digitize ( allocations , nonempty_ids , right = True ) if n_strata < goal_n_strata : warnings . warn ( "Failed to create {} strata" . format ( goal_n_strata ) ) return Strata ( allocations )
Stratify by binning the items based on their scores

    Parameters
    ----------
    scores : array-like, shape=(n_items,)
        ordered array of scores which quantify the classifier confidence for
        the items in the pool. High scores indicate a high confidence that
        the true label is a "1" (and vice versa for label "0").

    goal_n_strata : int or 'auto', optional, default 'auto'
        desired number of strata. If set to 'auto', the number is selected
        using the Freedman-Diaconis rule. Note that for the 'cum_sqrt_F' method
        this number is a goal -- the actual number of strata created may be
        less than the goal.

    method : {'cum_sqrt_F' or 'equal_size'}, optional, default 'cum_sqrt_F'
        stratification method to use. 'equal_size' aims to create s

    Other Parameters
    ----------------
    n_bins : int or 'auto', optional, default 'auto'
        specify the number of bins to use when estimating the distribution of
        the score function. This is used when ``goal_n_strata = 'auto'``
        and/or when ``method = 'cum_sqrt_F'``. If set to 'auto', the number is
        selected using the Freedman-Diaconis rule.

    Returns
    -------
    Strata instance