import numpy as np
import pandas as pd

# Load subset of the training data
X_train = pd.read_csv('../input/train.csv', skiprows=range(1,1000000), nrows=1000000, parse_dates=['click_time'])

# Show the head of the table
X_train.head()

#Feature Engineering
GROUP_BY_NEXT_CLICKS = [{'groupby': ['ip', 'app']}]

# Calculate the time to next click for each group
for spec in GROUP_BY_NEXT_CLICKS:

    # Name of new feature
    new_feature = '{}_nextClick'.format('_'.join(spec['groupby']))

    # Unique list of features to select
    all_features = spec['groupby'] + ['click_time']

    # Run calculation
    print(f">> Grouping by {spec['groupby']}, and saving time to next click in: {new_feature}")
    X_train[new_feature] = X_train[all_features].groupby(spec['groupby']).click_time.transform(lambda x: x.diff().shift(-1)).dt.seconds

X_train.head()

# Calculate information value
def calc_iv(df, feature, target, pr=False):
    """
    Set pr=True to enable printing of output.

    Output:
      * iv: float,
      * data: pandas.DataFrame
    """

    lst = []

    df[feature] = df[feature].fillna("NULL")

    for i in range(df[feature].nunique()):
        val = list(df[feature].unique())[i]
        lst.append([feature,                                                        # Variable
                    val,                                                            # Value
                    df[df[feature] == val].count()[feature],                        # All
                    df[(df[feature] == val) & (df[target] == 0)].count()[feature],  # Good (think: Fraud == 0)
                    df[(df[feature] == val) & (df[target] == 1)].count()[feature]]) # Bad (think: Fraud == 1)

    data = pd.DataFrame(lst, columns=['Variable', 'Value', 'All', 'Good', 'Bad'])

    data['Share'] = data['All'] / data['All'].sum()
    data['Bad Rate'] = data['Bad'] / data['All']
    data['Distribution Good'] = (data['All'] - data['Bad']) / (data['All'].sum() - data['Bad'].sum())
    data['Distribution Bad'] = data['Bad'] / data['Bad'].sum()
    data['WoE'] = np.log(data['Distribution Good'] / data['Distribution Bad'])

    data = data.replace({'WoE': {np.inf: 0, -np.inf: 0}})

    data['IV'] = data['WoE'] * (data['Distribution Good'] - data['Distribution Bad'])

    data = data.sort_values(by=['Variable', 'Value'], ascending=[True, True])
    data.index = range(len(data.index))

    if pr:
        print(data)
        print('IV = ', data['IV'].sum())


    iv = data['IV'].sum()
    # print(iv)

    return iv, data

df = X_train.sample(7777).copy()

iv, data = calc_iv(df, 'ip_app_nextClick', 'is_attributed')

iv

data.head()

#Hits
data[data['Bad Rate'] > 0.0]

import numpy as np
import pandas as pd

# Load subset of the training data
X_train = pd.read_csv('../input/train.csv', skiprows=range(1,1000000), nrows=1000000, parse_dates=['click_time'])

# Show the head of the table
X_train.head()

#Feature Engineering
GROUP_BY_NEXT_CLICKS = [{'groupby': ['ip', 'app']}]

# Calculate the time to next click for each group
for spec in GROUP_BY_NEXT_CLICKS:

    # Name of new feature
    new_feature = '{}_nextClick'.format('_'.join(spec['groupby']))

    # Unique list of features to select
    all_features = spec['groupby'] + ['click_time']

    # Run calculation
    print(f">> Grouping by {spec['groupby']}, and saving time to next click in: {new_feature}")
    X_train[new_feature] = X_train[all_features].groupby(spec['groupby']).click_time.transform(lambda x: x.diff().shift(-1)).dt.seconds

X_train.head()

# Calculate information value
def calc_iv(df, feature, target, pr=False):
    """
    Set pr=True to enable printing of output.

    Output:
      * iv: float,
      * data: pandas.DataFrame
    """

    lst = []

    df[feature] = df[feature].fillna("NULL")

    for i in range(df[feature].nunique()):
        val = list(df[feature].unique())[i]
        lst.append([feature,                                                        # Variable
                    val,                                                            # Value
                    df[df[feature] == val].count()[feature],                        # All
                    df[(df[feature] == val) & (df[target] == 0)].count()[feature],  # Good (think: Fraud == 0)
                    df[(df[feature] == val) & (df[target] == 1)].count()[feature]]) # Bad (think: Fraud == 1)

    data = pd.DataFrame(lst, columns=['Variable', 'Value', 'All', 'Good', 'Bad'])

    data['Share'] = data['All'] / data['All'].sum()
    data['Bad Rate'] = data['Bad'] / data['All']
    data['Distribution Good'] = (data['All'] - data['Bad']) / (data['All'].sum() - data['Bad'].sum())
    data['Distribution Bad'] = data['Bad'] / data['Bad'].sum()
    data['WoE'] = np.log(data['Distribution Good'] / data['Distribution Bad'])

    data = data.replace({'WoE': {np.inf: 0, -np.inf: 0}})

    data['IV'] = data['WoE'] * (data['Distribution Good'] - data['Distribution Bad'])

    data = data.sort_values(by=['Variable', 'Value'], ascending=[True, True])
    data.index = range(len(data.index))

    if pr:
        print(data)
        print('IV = ', data['IV'].sum())


    iv = data['IV'].sum()
    # print(iv)

    return iv, data

df = X_train.sample(7777).copy()

iv, data = calc_iv(df, 'ip_app_nextClick', 'is_attributed')

iv

data.head()

#Hits
data[data['Bad Rate'] > 0.0]

# Load subset of the training data # Show the head of the table #Feature Engineering # Calculate the time to next click for each group # Name of new feature # Unique list of features to select # Run calculation # Calculate information value Set pr=True to enable printing of output.
 
     Output:
       * iv: float,
       * data: pandas.DataFrame # Variable # Value # All # Good (think: Fraud == 0) # Bad (think: Fraud == 1) # print(iv) #Hits