<reponame>KazukiOnodera/Microsoft-Malware-Prediction
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Feb  4 18:10:01 2019

@author: Kazuki

cp LOG/imp_803_holdout.py.csv LOG/imp_used_0204-1.csv


"""

import numpy as np
import pandas as pd
import os, gc
from glob import glob
from tqdm import tqdm

import sys
sys.path.append(f'/home/{os.environ.get("USER")}/PythonLibrary')
import lgbextension as ex
import lightgbm as lgb
from multiprocessing import cpu_count

from sklearn.metrics import roc_auc_score

import utils , utils_cat
utils.start(__file__)
#==============================================================================

SUBMIT_FILE_PATH = '../output/0204-1.csv.gz'

COMMENT = 'f001 ~ f012'

EXE_SUBMIT = True

#SEED = np.random.randint(9999)
SEED = 71
print('SEED:', SEED)


LOOP1 = 2
LOOP2 = 10

param = {
         'objective': 'binary',
         'metric': 'None',
         
         'learning_rate': 0.1,
         'max_depth': 5,
         'num_leaves': 2**6 -1,
         'max_bin': 127,
         
         'min_child_weight': 10,
         'min_data_in_leaf': 150,
         'reg_lambda': 0.5,  # L2 regularization term on weights.
         'reg_alpha': 0.5,  # L1 regularization term on weights.
         
         'colsample_bytree': 0.8,
         'subsample': 0.8,
#         'nthread': 32,
         'nthread': cpu_count(),
         'bagging_freq': 1,
         'verbose':-1,
         }

ESR = 100
VERBOSE_EVAL = 50

TRAIN_TH = 0.6
VALID_TH = 0.8

IMP_FILE = 'LOG/imp_used_0204-1.csv'
#USE_PREF = ['f001', 'f002', 'f003', 'f009', 'f010', 'f011', 'f012']

RESULT_DICT = {}
RESULT_DICT['file'] = SUBMIT_FILE_PATH


# =============================================================================
# load
# =============================================================================
imp = pd.read_csv(IMP_FILE)
COL = imp[imp['total']>0].feature.tolist()
#COL = imp.feature.tolist()

files_tr = sorted(glob('../data/train_f*.f'))

# USE_PREF
#li = []
#for i in files_tr:
#    for j in USE_PREF:
#        if j in i:
#            li.append(i)
#            break
#files_tr = li

[print(i,f) for i,f in enumerate(files_tr)]

X_train = pd.concat([
                pd.read_feather(f) for f in tqdm(files_tr, mininterval=60)
               ], axis=1)[COL]
y_train = utils.load_target()['HasDetections']

adv = pd.read_csv('../data/oof_802_adv.py.csv').iloc[:8921483].oof
adv_th = adv.quantile(VALID_TH)

X_valid = X_train[adv>adv.quantile(VALID_TH)]
y_valid = y_train[adv>adv.quantile(VALID_TH)]

X_train = X_train[adv<=adv.quantile(TRAIN_TH)]
y_train = y_train[adv<=adv.quantile(TRAIN_TH)]

if X_train.columns.duplicated().sum()>0:
    raise Exception(f'duplicated!: { X_train.columns[X_train.columns.duplicated()] }')
print('no dup :) ')
print(f'X_train.shape {X_train.shape}')
print(f'X_valid.shape {X_valid.shape}')

gc.collect()

CAT = list( set(X_train.columns)&set(utils_cat.ALL))
print(f'CAT: {CAT}')


#COL = X_train.columns.tolist()

RESULT_DICT['feature size'] = len(COL)
RESULT_DICT['category feature size'] = len(CAT)


# =============================================================================
# hold out
# =============================================================================


dtrain = lgb.Dataset(X_train, y_train.values, 
                     categorical_feature=CAT, 
                     free_raw_data=False)

dvalid = lgb.Dataset(X_valid, y_valid.values, 
                     categorical_feature=CAT, 
                     free_raw_data=False)
gc.collect()



model_all = []
y_preds = []
score_list = []
nround_mean = 0
for i in range(LOOP1):
    param['seed'] = np.random.randint(9999)
    model = lgb.train(params=param, train_set=dtrain, num_boost_round=9999, 
                      valid_sets=[dtrain, dvalid], 
                      valid_names=['train','valid'], 
                      feval=ex.eval_auc,
                      categorical_feature=CAT, 
                      early_stopping_rounds=ESR,
                      verbose_eval=VERBOSE_EVAL)
    y_preds.append(pd.Series(model.predict(X_valid)))
    model_all.append(model)
    nround_mean += model.best_iteration
    score_list.append(model.best_score['valid']['auc'])

nround_mean /= LOOP1

imp = ex.getImp(model_all)
imp['split'] /= imp['split'].max()
imp['gain'] /= imp['gain'].max()
imp['total'] = imp['split'] + imp['gain']
imp.sort_values('total', ascending=False, inplace=True)
imp.reset_index(drop=True, inplace=True)


for i,y_pred in enumerate(y_preds):
    y_pred = y_pred.rank()
    if i==0:
        oof = y_pred
    else:
        oof += y_pred
oof /= len(y_preds)
oof /= oof.max()


imp.to_csv(f'LOG/imp_{__file__}.csv', index=False)
pd.DataFrame(oof, columns=['oof']).to_csv(f'../data/oof_{__file__}.csv', index=False)

utils.savefig_imp(imp, f'LOG/imp_{__file__}.png', x='total')

RESULT_DICT['nfold'] = '-'
RESULT_DICT['loop'] = LOOP2
RESULT_DICT['eta'] = param['learning_rate']
RESULT_DICT['TRAIN_TH'] = TRAIN_TH
RESULT_DICT['VALID_TH'] = VALID_TH
RESULT_DICT['nround mean'] = nround_mean
RESULT_DICT['cv AUC mean'] = round(np.mean(score_list), 5)
RESULT_DICT['oof AUC'] = round(roc_auc_score(y_valid, oof), 5)




# =============================================================================
# all sample
# =============================================================================

X_train = pd.concat([
                pd.read_feather(f) for f in tqdm(files_tr, mininterval=60)
               ], axis=1)[COL]
y_train = utils.load_target()['HasDetections']

nround = int(nround_mean * 1.7)

dtrain = lgb.Dataset(X_train, y_train, free_raw_data=False)

del X_train, y_train, X_valid, y_valid, dvalid; gc.collect()

model_all = []
for i in range(LOOP2):
    print('building', i)
    param['seed'] = np.random.randint(9999)
    model = lgb.train(params=param, train_set=dtrain, num_boost_round=nround, 
                      feval=ex.eval_auc,
                      categorical_feature=CAT,
                      verbose_eval=VERBOSE_EVAL)
    model_all.append(model)

del dtrain; gc.collect()


# =============================================================================
# test
# =============================================================================


files_te = sorted(glob('../data/test_f*.f'))

# USE_PREF
#li = []
#for i in files_te:
#    for j in USE_PREF:
#        if j in i:
#            li.append(i)
#            break
#files_te = li

[print(i,f) for i,f in enumerate(files_te)]

X_test = pd.concat([
                pd.read_feather(f) for f in tqdm(files_te, mininterval=60)
               ], axis=1)[COL]


if X_test.columns.duplicated().sum()>0:
    raise Exception(f'duplicated!: { X_test.columns[X_test.columns.duplicated()] }')
print('no dup :) ')
print(f'X_test.shape {X_test.shape}')


for i,model in enumerate(tqdm(model_all)):
    gc.collect()
    y_pred = pd.Series(model.predict(X_test)).rank()
    if i==0:
        y_pred_all = y_pred
    else:
        y_pred_all += y_pred

y_pred_all /= len(model_all)
y_pred_all /= y_pred_all.max()

sub = pd.read_csv('../input/sample_submission.csv.zip')
sub['HasDetections'] = y_pred_all.values


print('corr with best')
sub_best = pd.read_csv(utils.SUB_BEST)
print(sub['HasDetections'].corr( sub_best['HasDetections'], method='spearman') )

print('corr with all subs')
for file in utils.SUB_CORR:
    tmp = pd.read_csv(file)
    corr = sub['HasDetections'].corr( tmp['HasDetections'], method='spearman')
    print(f'{file}: {corr:.5f}')



print("""
# =============================================================================
# write down these info to benchmark.xlsx
# =============================================================================
""")

[print(f'{k:<25}: {RESULT_DICT[k]}') for k in RESULT_DICT]

print("""
# =============================================================================
""")




# save
sub.to_csv(SUBMIT_FILE_PATH, index=False, compression='gzip')


# =============================================================================
# submission
# =============================================================================
if EXE_SUBMIT:
    print('submit')
    utils.submit(SUBMIT_FILE_PATH, COMMENT)




#==============================================================================
utils.end(__file__)
#utils.stop_instance()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Feb  4 18:10:01 2019

@author: Kazuki

cp LOG/imp_803_holdout.py.csv LOG/imp_used_0204-1.csv


"""

import numpy as np
import pandas as pd
import os, gc
from glob import glob
from tqdm import tqdm

import sys
sys.path.append(f'/home/{os.environ.get("USER")}/PythonLibrary')
import lgbextension as ex
import lightgbm as lgb
from multiprocessing import cpu_count

from sklearn.metrics import roc_auc_score

import utils , utils_cat
utils.start(__file__)
#==============================================================================

SUBMIT_FILE_PATH = '../output/0204-1.csv.gz'

COMMENT = 'f001 ~ f012'

EXE_SUBMIT = True

#SEED = np.random.randint(9999)
SEED = 71
print('SEED:', SEED)


LOOP1 = 2
LOOP2 = 10

param = {
         'objective': 'binary',
         'metric': 'None',
         
         'learning_rate': 0.1,
         'max_depth': 5,
         'num_leaves': 2**6 -1,
         'max_bin': 127,
         
         'min_child_weight': 10,
         'min_data_in_leaf': 150,
         'reg_lambda': 0.5,  # L2 regularization term on weights.
         'reg_alpha': 0.5,  # L1 regularization term on weights.
         
         'colsample_bytree': 0.8,
         'subsample': 0.8,
#         'nthread': 32,
         'nthread': cpu_count(),
         'bagging_freq': 1,
         'verbose':-1,
         }

ESR = 100
VERBOSE_EVAL = 50

TRAIN_TH = 0.6
VALID_TH = 0.8

IMP_FILE = 'LOG/imp_used_0204-1.csv'
#USE_PREF = ['f001', 'f002', 'f003', 'f009', 'f010', 'f011', 'f012']

RESULT_DICT = {}
RESULT_DICT['file'] = SUBMIT_FILE_PATH


# =============================================================================
# load
# =============================================================================
imp = pd.read_csv(IMP_FILE)
COL = imp[imp['total']>0].feature.tolist()
#COL = imp.feature.tolist()

files_tr = sorted(glob('../data/train_f*.f'))

# USE_PREF
#li = []
#for i in files_tr:
#    for j in USE_PREF:
#        if j in i:
#            li.append(i)
#            break
#files_tr = li

[print(i,f) for i,f in enumerate(files_tr)]

X_train = pd.concat([
                pd.read_feather(f) for f in tqdm(files_tr, mininterval=60)
               ], axis=1)[COL]
y_train = utils.load_target()['HasDetections']

adv = pd.read_csv('../data/oof_802_adv.py.csv').iloc[:8921483].oof
adv_th = adv.quantile(VALID_TH)

X_valid = X_train[adv>adv.quantile(VALID_TH)]
y_valid = y_train[adv>adv.quantile(VALID_TH)]

X_train = X_train[adv<=adv.quantile(TRAIN_TH)]
y_train = y_train[adv<=adv.quantile(TRAIN_TH)]

if X_train.columns.duplicated().sum()>0:
    raise Exception(f'duplicated!: { X_train.columns[X_train.columns.duplicated()] }')
print('no dup :) ')
print(f'X_train.shape {X_train.shape}')
print(f'X_valid.shape {X_valid.shape}')

gc.collect()

CAT = list( set(X_train.columns)&set(utils_cat.ALL))
print(f'CAT: {CAT}')


#COL = X_train.columns.tolist()

RESULT_DICT['feature size'] = len(COL)
RESULT_DICT['category feature size'] = len(CAT)


# =============================================================================
# hold out
# =============================================================================


dtrain = lgb.Dataset(X_train, y_train.values, 
                     categorical_feature=CAT, 
                     free_raw_data=False)

dvalid = lgb.Dataset(X_valid, y_valid.values, 
                     categorical_feature=CAT, 
                     free_raw_data=False)
gc.collect()



model_all = []
y_preds = []
score_list = []
nround_mean = 0
for i in range(LOOP1):
    param['seed'] = np.random.randint(9999)
    model = lgb.train(params=param, train_set=dtrain, num_boost_round=9999, 
                      valid_sets=[dtrain, dvalid], 
                      valid_names=['train','valid'], 
                      feval=ex.eval_auc,
                      categorical_feature=CAT, 
                      early_stopping_rounds=ESR,
                      verbose_eval=VERBOSE_EVAL)
    y_preds.append(pd.Series(model.predict(X_valid)))
    model_all.append(model)
    nround_mean += model.best_iteration
    score_list.append(model.best_score['valid']['auc'])

nround_mean /= LOOP1

imp = ex.getImp(model_all)
imp['split'] /= imp['split'].max()
imp['gain'] /= imp['gain'].max()
imp['total'] = imp['split'] + imp['gain']
imp.sort_values('total', ascending=False, inplace=True)
imp.reset_index(drop=True, inplace=True)


for i,y_pred in enumerate(y_preds):
    y_pred = y_pred.rank()
    if i==0:
        oof = y_pred
    else:
        oof += y_pred
oof /= len(y_preds)
oof /= oof.max()


imp.to_csv(f'LOG/imp_{__file__}.csv', index=False)
pd.DataFrame(oof, columns=['oof']).to_csv(f'../data/oof_{__file__}.csv', index=False)

utils.savefig_imp(imp, f'LOG/imp_{__file__}.png', x='total')

RESULT_DICT['nfold'] = '-'
RESULT_DICT['loop'] = LOOP2
RESULT_DICT['eta'] = param['learning_rate']
RESULT_DICT['TRAIN_TH'] = TRAIN_TH
RESULT_DICT['VALID_TH'] = VALID_TH
RESULT_DICT['nround mean'] = nround_mean
RESULT_DICT['cv AUC mean'] = round(np.mean(score_list), 5)
RESULT_DICT['oof AUC'] = round(roc_auc_score(y_valid, oof), 5)




# =============================================================================
# all sample
# =============================================================================

X_train = pd.concat([
                pd.read_feather(f) for f in tqdm(files_tr, mininterval=60)
               ], axis=1)[COL]
y_train = utils.load_target()['HasDetections']

nround = int(nround_mean * 1.7)

dtrain = lgb.Dataset(X_train, y_train, free_raw_data=False)

del X_train, y_train, X_valid, y_valid, dvalid; gc.collect()

model_all = []
for i in range(LOOP2):
    print('building', i)
    param['seed'] = np.random.randint(9999)
    model = lgb.train(params=param, train_set=dtrain, num_boost_round=nround, 
                      feval=ex.eval_auc,
                      categorical_feature=CAT,
                      verbose_eval=VERBOSE_EVAL)
    model_all.append(model)

del dtrain; gc.collect()


# =============================================================================
# test
# =============================================================================


files_te = sorted(glob('../data/test_f*.f'))

# USE_PREF
#li = []
#for i in files_te:
#    for j in USE_PREF:
#        if j in i:
#            li.append(i)
#            break
#files_te = li

[print(i,f) for i,f in enumerate(files_te)]

X_test = pd.concat([
                pd.read_feather(f) for f in tqdm(files_te, mininterval=60)
               ], axis=1)[COL]


if X_test.columns.duplicated().sum()>0:
    raise Exception(f'duplicated!: { X_test.columns[X_test.columns.duplicated()] }')
print('no dup :) ')
print(f'X_test.shape {X_test.shape}')


for i,model in enumerate(tqdm(model_all)):
    gc.collect()
    y_pred = pd.Series(model.predict(X_test)).rank()
    if i==0:
        y_pred_all = y_pred
    else:
        y_pred_all += y_pred

y_pred_all /= len(model_all)
y_pred_all /= y_pred_all.max()

sub = pd.read_csv('../input/sample_submission.csv.zip')
sub['HasDetections'] = y_pred_all.values


print('corr with best')
sub_best = pd.read_csv(utils.SUB_BEST)
print(sub['HasDetections'].corr( sub_best['HasDetections'], method='spearman') )

print('corr with all subs')
for file in utils.SUB_CORR:
    tmp = pd.read_csv(file)
    corr = sub['HasDetections'].corr( tmp['HasDetections'], method='spearman')
    print(f'{file}: {corr:.5f}')



print("""
# =============================================================================
# write down these info to benchmark.xlsx
# =============================================================================
""")

[print(f'{k:<25}: {RESULT_DICT[k]}') for k in RESULT_DICT]

print("""
# =============================================================================
""")




# save
sub.to_csv(SUBMIT_FILE_PATH, index=False, compression='gzip')


# =============================================================================
# submission
# =============================================================================
if EXE_SUBMIT:
    print('submit')
    utils.submit(SUBMIT_FILE_PATH, COMMENT)




#==============================================================================
utils.end(__file__)
#utils.stop_instance()
#!/usr/bin/env python3 # -*- coding: utf-8 -*- Created on Mon Feb  4 18:10:01 2019  @author: Kazuki  cp LOG/imp_803_holdout.py.csv LOG/imp_used_0204-1.csv #============================================================================== #SEED = np.random.randint(9999) # L2 regularization term on weights. # L1 regularization term on weights. #         'nthread': 32, #USE_PREF = ['f001', 'f002', 'f003', 'f009', 'f010', 'f011', 'f012'] # ============================================================================= # load # ============================================================================= #COL = imp.feature.tolist() # USE_PREF #li = [] #for i in files_tr: #    for j in USE_PREF: #        if j in i: #            li.append(i) #            break #files_tr = li #COL = X_train.columns.tolist() # ============================================================================= # hold out # ============================================================================= # ============================================================================= # all sample # ============================================================================= # ============================================================================= # test # ============================================================================= # USE_PREF #li = [] #for i in files_te: #    for j in USE_PREF: #        if j in i: #            li.append(i) #            break #files_te = li # ============================================================================= # write down these info to benchmark.xlsx # ============================================================================= # ============================================================================= # save # ============================================================================= # submission # ============================================================================= #============================================================================== #utils.stop_instance()