def map_fit(interface, state, label, inp):
    """
    Function counts occurrences of feature values for every row in given data chunk. For continuous features it returns
    number of values and it calculates mean and variance for every feature.
    For discrete features it counts occurrences of labels and values for every feature. It returns occurrences of pairs:
    label, feature index, feature values.
    """
    import numpy as np
    combiner = {}  # combiner used for joining of intermediate pairs
    out = interface.output(0)  # all outputted pairs have the same output label

    for row in inp:  # for every row in data chunk
        row = row.strip().split(state["delimiter"])  # split row
        if len(row) > 1:  # check if row is empty
            for i, j in enumerate(state["X_indices"]):  # for defined features
                if row[j] not in state["missing_vals"]:  # check missing values
                    # creates a pair - label, feature index
                    pair = row[state["y_index"]] + state["delimiter"] + str(j)

                    if state["X_meta"][i] == "c":  # continuous features
                        if pair in combiner:
                            # convert to float and store value
                            combiner[pair].append(np.float32(row[j]))
                        else:
                            combiner[pair] = [np.float32(row[j])]

                    else:  # discrete features
                        # add feature value to pair
                        pair += state["delimiter"] + row[j]
                        # increase counts of current pair
                        combiner[pair] = combiner.get(pair, 0) + 1

                    # increase label counts
                    combiner[row[state["y_index"]]] = combiner.get(row[state["y_index"]], 0) + 1

    for k, v in combiner.iteritems():  # all pairs in combiner are output
        if len(k.split(state["delimiter"])) == 2:  # continous features
            # number of elements, partial mean and variance
            out.add(k, (np.size(v), np.mean(v, dtype=np.float32), np.var(v, dtype=np.float32)))
        else:  # discrete features and labels
            out.add(k, v)
def map_fit ( interface , state , label , inp ) : import numpy as np combiner = { } out = interface . output ( 0 ) for row in inp : row = row . strip ( ) . split ( state [ "delimiter" ] ) if len ( row ) > 1 : for i , j in enumerate ( state [ "X_indices" ] ) : if row [ j ] not in state [ "missing_vals" ] : pair = row [ state [ "y_index" ] ] + state [ "delimiter" ] + str ( j ) if state [ "X_meta" ] [ i ] == "c" : if pair in combiner : combiner [ pair ] . append ( np . float32 ( row [ j ] ) ) else : combiner [ pair ] = [ np . float32 ( row [ j ] ) ] else : pair += state [ "delimiter" ] + row [ j ] combiner [ pair ] = combiner . get ( pair , 0 ) + 1 combiner [ row [ state [ "y_index" ] ] ] = combiner . get ( row [ state [ "y_index" ] ] , 0 ) + 1 for k , v in combiner . iteritems ( ) : if len ( k . split ( state [ "delimiter" ] ) ) == 2 : out . add ( k , ( np . size ( v ) , np . mean ( v , dtype = np . float32 ) , np . var ( v , dtype = np . float32 ) ) ) else : out . add ( k , v )
Function counts occurrences of feature values for every row in given data chunk. For continuous features it returns
    number of values and it calculates mean and variance for every feature.
    For discrete features it counts occurrences of labels and values for every feature. It returns occurrences of pairs:
    label, feature index, feature values.