src/nba_processing/nba_RHGN_pre_processing.py

import numpy as np
import pandas as pd
import dgl
import torch
from fainress_component import disparate_impact_remover, reweighting, sample
from utils import apply_bin_columns, apply_cat_columns
import fastText

def nba_RHGN_pre_process(df, dataset_user_id_name, sens_attr, label, onehot_bin_columns, onehot_cat_columns, debaising_approach=None):

    if onehot_bin_columns != None:
        df = apply_bin_columns(df, onehot_bin_columns)
    
    if onehot_cat_columns != None:
        df = apply_cat_columns(df, onehot_cat_columns)

    # nba case
    if -1 in df[label].unique():
        df[label] = df[label].replace(-1, 0)

    df = df.astype({'user_id': 'str'}, copy=False)
    df = df.astype({'AGE':'str', 'MP':'str', 'FG':'str'}, copy=False)


    if debaising_approach != None:
        if debaising_approach == 'disparate_impact_remover':
            df = disparate_impact_remover(df, sens_attr, label)
        elif debaising_approach == 'reweighting':
            df = reweighting(df, sens_attr, label)
        elif debaising_approach == 'sample':
            df = sample(df, sens_attr, label)
            df = df.drop_duplicates()


    if debaising_approach == 'disparate_impact_remover' or debaising_approach == 'reweighting':
        df.AGE = df.AGE.astype(int)
        df.country = df.country.astype(int)
        df.SALARY = df.SALARY.astype(int)

        df['user_id'] = pd.to_numeric(df['user_id'])
        df = df.astype({'user_id': int})

        df.AGE = df.AGE.astype(str)
        df.MP = df.MP.astype(str)
        df.FG = df.FG.astype(str)

    
    user_dic = {k: v for v, k in enumerate(df.user_id.drop_duplicates())}
    age_dic = {k: v for v, k in enumerate(df.AGE.drop_duplicates())}
    mp_dic = {k: v for v, k in enumerate(df.MP.drop_duplicates())}
    fg_dic = {k: v for v, k in enumerate(df.FG.drop_duplicates())}

    item_dic = {}
    c1, c2, c3=[], [], []
    
    if debaising_approach == 'sample':
        for i, row in df.iterrows():
            #print(i)
            c1_1 = df.at[i, 'AGE']
            #print(c1_1)
            if isinstance(c1_1, str):
                c1.append(age_dic[c1_1])
            else:
                c1.append(age_dic[c1_1.iloc[0]])
                
            c2_2 = df.at[i, 'MP']
            if isinstance(c2_2, str):
                c2.append(mp_dic[c2_2])
            else:
                c2.append(mp_dic[c2_2.iloc[0]])
                
            c3_3 = df.at[i, 'FG']
            if isinstance(c3_3, str):
                c3.append(fg_dic[c3_3])
            else:
                c3.append(fg_dic[c3_3.iloc[0]])

    elif debaising_approach == 'disparate_impact_remover' or debaising_approach == 'reweighting':
        for i in range(len(df)):
            c1.append(age_dic[df['AGE'].iloc[i]])
            c2.append(mp_dic[df['MP'].iloc[i]])
            c3.append(fg_dic[df['FG'].iloc[i]])
    else:
        for i in range(len(df)):
            c1.append(age_dic[df.at[i, 'AGE']])
            c2.append(mp_dic[df.at[i, 'MP']])
            c3.append(fg_dic[df.at[i, 'FG']])
        
        
    print(min(c1), min(c2), min(c3))
    print(len(age_dic), len(mp_dic), len(fg_dic))

    has_user = [user_dic[user] for user in df.user_id]
    is_made_by_user = [mp_dic[item] for item in df.MP]


    data_dict = {
        ("user", "has", "item"): (torch.tensor(has_user), torch.tensor(is_made_by_user)),
        ("item", "is_made_by", "user"): (torch.tensor(is_made_by_user), torch.tensor(has_user))
    }


    G = dgl.heterograph(data_dict)

    model = fasttext.load_model('../cc.zh.200.bin')

    temp1 = {k: model.get_sentence_vector(v) for v, k in age_dic.items()}
    cid1_feature = torch.tensor([temp1[k] for _, k in age_dic.items()])

    temp2 = {k: model.get_sentence_vector(v) for v, k in mp_dic.items()}
    cid2_feature = torch.tensor([temp2[k] for _, k in mp_dic.items()])
 

    temp3 = {k: model.get_sentence_vector(v) for v, k in fg_dic.items()}
    cid3_feature = torch.tensor([temp3[k] for _, k in fg_dic.items()])


    uid2id = {num: i for i, num in enumerate(df[dataset_user_id_name])}

    df_user = col_map(df, dataset_user_id_name, uid2id)
    user_label = label_map(df_user, df_user.columns[1:])

    # todo let the user define what to have in the graph?
    label_age = user_label.AGE
    label_height = user_label.player_height
    label_weight = user_label.player_weight
    label_country = user_label.country
    #label_teams = user_label.teams
    label_salary = user_label.SALARY

    G.nodes['user'].data['age'] = torch.tensor(label_age[:G.number_of_nodes('user')].values)
    G.nodes['user'].data['height'] = torch.tensor(label_height[:G.number_of_nodes('user')].values)
    G.nodes['user'].data['weight'] = torch.tensor(label_weight[:G.number_of_nodes('user')].values)
    G.nodes['user'].data['country'] = torch.tensor(label_country[:G.number_of_nodes('user')].values)
    #G.nodes['user'].data['teams'] = torch.tensor(label_teams[:G.number_of_nodes('user')])
    G.nodes['user'].data['SALARY'] = torch.tensor(label_salary[:G.number_of_nodes('user')].values)

    G.nodes['item'].data['cid1'] = torch.tensor(c1[:G.number_of_nodes('item')])
    G.nodes['item'].data['cid2'] = torch.tensor(c2[:G.number_of_nodes('item')])
    G.nodes['item'].data['cid3'] = torch.tensor(c3[:G.number_of_nodes('item')])

    
    print(G)
    print(cid1_feature.shape)
    print(cid2_feature.shape)
    print(cid3_feature.shape)


    return G, cid1_feature, cid2_feature, cid3_feature


def col_map(df, col, num2id):
    df[[col]] = df[[col]].applymap(lambda x: num2id[x])
    return df

def label_map(label_df, label_list):
    for label in label_list:
        label2id = {num: i for i, num in enumerate(pd.unique(label_df[label]))}
        label_df = col_map(label_df, label, label2id)
    return label_df