def info_gain_nominal(x, y, separate_max):
    """
    Function calculates information gain for discrete features. If feature is continuous it is firstly discretized.

    x: numpy array - numerical or discrete feature
    y: numpy array - labels
    ft: string - feature type ("c" - continuous, "d" - discrete)
    split_fun: function - function for discretization of numerical features
    """
    x_vals = np.unique(x)  # unique values
    if len(x_vals) < 3:  # if there is just one unique value
        return None
    y_dist = Counter(y)  # label distribution
    h_y = h(y_dist.values())  # class entropy

    # calculate distributions and splits in accordance with feature type

    dist, splits = nominal_splits(x, y, x_vals, y_dist, separate_max)

    indices, repeat = (range(1, len(dist)), 1) if len(dist) < 50 else (range(1, len(dist), len(dist) / 10), 3)
    interval = len(dist) / 10

    max_ig, max_i, iteration = 0, 1, 0
    while iteration < repeat:
        for i in indices:
            dist0 = np.sum([el for el in dist[:i]])  # iter 0: take first distribution
            dist1 = np.sum([el for el in dist[i:]])  # iter 0: take the other distributions without first
            coef = np.true_divide([np.sum(dist0.values()), np.sum(dist1.values())], len(y))
            ig = h_y - np.dot(coef, [h(dist0.values()), h(dist1.values())])  # calculate information gain
            if ig > max_ig:
                max_ig, max_i = ig, i  # store index and value of maximal information gain
        iteration += 1
        if repeat > 1:
            interval = int(interval * 0.5)
            if max_i in indices and interval > 0:
                middle_index = indices.index(max_i)
            else:
                break
            min_index = middle_index if middle_index == 0 else middle_index - 1
            max_index = middle_index if middle_index == len(indices) - 1 else middle_index + 1
            indices = range(indices[min_index], indices[max_index], interval)

    # store splits of maximal information gain in accordance with feature type
    return float(max_ig), [splits[:max_i], splits[max_i:]]
def info_gain_nominal ( x , y , separate_max ) : x_vals = np . unique ( x ) if len ( x_vals ) < 3 : return None y_dist = Counter ( y ) h_y = h ( y_dist . values ( ) ) dist , splits = nominal_splits ( x , y , x_vals , y_dist , separate_max ) indices , repeat = ( range ( 1 , len ( dist ) ) , 1 ) if len ( dist ) < 50 else ( range ( 1 , len ( dist ) , len ( dist ) / 10 ) , 3 ) interval = len ( dist ) / 10 max_ig , max_i , iteration = 0 , 1 , 0 while iteration < repeat : for i in indices : dist0 = np . sum ( [ el for el in dist [ : i ] ] ) dist1 = np . sum ( [ el for el in dist [ i : ] ] ) coef = np . true_divide ( [ np . sum ( dist0 . values ( ) ) , np . sum ( dist1 . values ( ) ) ] , len ( y ) ) ig = h_y - np . dot ( coef , [ h ( dist0 . values ( ) ) , h ( dist1 . values ( ) ) ] ) if ig > max_ig : max_ig , max_i = ig , i iteration += 1 if repeat > 1 : interval = int ( interval * 0.5 ) if max_i in indices and interval > 0 : middle_index = indices . index ( max_i ) else : break min_index = middle_index if middle_index == 0 else middle_index - 1 max_index = middle_index if middle_index == len ( indices ) - 1 else middle_index + 1 indices = range ( indices [ min_index ] , indices [ max_index ] , interval ) return float ( max_ig ) , [ splits [ : max_i ] , splits [ max_i : ] ]
Function calculates information gain for discrete features. If feature is continuous it is firstly discretized.

    x: numpy array - numerical or discrete feature
    y: numpy array - labels
    ft: string - feature type ("c" - continuous, "d" - discrete)
    split_fun: function - function for discretization of numerical features