<filename>codes/train.py
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
@File    :   train.py
@Contact :   <EMAIL>
@License :   (C)Copyright 2021-2022 PowerLZY

@Modify Time      @Author               @Version    @Desciption
------------      -------               --------    -----------
2021-10-04 15:03   PowerLZY&yuan_mes       1.0      保存训练模型，特征选择模型和单特征类别权重
'''
import warnings
import os
import sys
import joblib
import numpy as np
import pandas as pd

curPath = os.path.abspath(os.path.dirname("__file__"))
rootPath = os.path.split(curPath)[0]
sys.path.append(rootPath)

from sklearn.ensemble import ExtraTreesClassifier
from sklearn.feature_selection import SelectFromModel
from sklearn.utils import class_weight
from xgboost import XGBClassifier

from codes.utils import load_data
from codes.feature_engineering import feature_engineering
from codes.model import Model


warnings.filterwarnings('ignore')


def train_model(train_data_dict, train_labels, inter_path):

    model_A = XGBClassifier(
        objective='multi:softprob',
        num_class=10,
        max_depth=6,
        n_estimators=90,
        learning_rate=0.1,
        eval_metric='mlogloss',
        use_label_encoder=False
    )

    classes_weights = class_weight.compute_sample_weight(
        class_weight='balanced',
        y=train_labels
    )

    labels_loss = pd.DataFrame()
    print(f"------------------------ 开始训练 ------------------------")
    for name, train_data in train_data_dict.items():

        if name in ['words_1000', 'words_300', 'ember_section_ins_words', 'ember_section_ins_semantic']:
            # 使用了TF-IDF的特征做特征选择
            selector = SelectFromModel(estimator=ExtraTreesClassifier(n_estimators=200)).fit(train_data, train_labels,
                                                                                        sample_weight=classes_weights)

            joblib.dump(selector, open(f"{inter_path}/models/select_model_{name}.pth", "wb"))
            train_data = selector.transform(train_data)

        clf = Model(model_A, train_data, train_labels, name, inter_path, labels_loss)
        clf.Fit()

        labels_loss[name] = clf.get_class_weight()

    labels_loss[np.isnan(labels_loss)] = 0
    labels_loss[labels_loss < 0] = 0
    labels_loss.to_csv(f"{inter_path}/feature/labels_loss.csv", index=False)
    print(f"------------------------ 训练完成 ------------------------")

if __name__ == '__main__':

    inter_path = '../data/user_data'
    data_path = '../data/raw_data'
    feature_list = ['ember', 'section', 'imports', 'exports', 'words_1000', 'semantic', 'ember_section_ins_words', 'ember_section_ins_semantic']

    print(f"------------------------ 训练集特征工程 ------------------------")
    feature_engineering("train", data_path, inter_path)
    train_data_dict = load_data('train', feature_list, inter_path)

    train_lab_path = f"{inter_path}/train_y.npy"
    train_y = np.load(train_lab_path)

    train_model(train_data_dict, train_y, inter_path)

<filename>codes/train.py
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
@File    :   train.py
@Contact :   <EMAIL>
@License :   (C)Copyright 2021-2022 PowerLZY

@Modify Time      @Author               @Version    @Desciption
------------      -------               --------    -----------
2021-10-04 15:03   PowerLZY&yuan_mes       1.0      保存训练模型，特征选择模型和单特征类别权重
'''
import warnings
import os
import sys
import joblib
import numpy as np
import pandas as pd

curPath = os.path.abspath(os.path.dirname("__file__"))
rootPath = os.path.split(curPath)[0]
sys.path.append(rootPath)

from sklearn.ensemble import ExtraTreesClassifier
from sklearn.feature_selection import SelectFromModel
from sklearn.utils import class_weight
from xgboost import XGBClassifier

from codes.utils import load_data
from codes.feature_engineering import feature_engineering
from codes.model import Model


warnings.filterwarnings('ignore')


def train_model(train_data_dict, train_labels, inter_path):

    model_A = XGBClassifier(
        objective='multi:softprob',
        num_class=10,
        max_depth=6,
        n_estimators=90,
        learning_rate=0.1,
        eval_metric='mlogloss',
        use_label_encoder=False
    )

    classes_weights = class_weight.compute_sample_weight(
        class_weight='balanced',
        y=train_labels
    )

    labels_loss = pd.DataFrame()
    print(f"------------------------ 开始训练 ------------------------")
    for name, train_data in train_data_dict.items():

        if name in ['words_1000', 'words_300', 'ember_section_ins_words', 'ember_section_ins_semantic']:
            # 使用了TF-IDF的特征做特征选择
            selector = SelectFromModel(estimator=ExtraTreesClassifier(n_estimators=200)).fit(train_data, train_labels,
                                                                                        sample_weight=classes_weights)

            joblib.dump(selector, open(f"{inter_path}/models/select_model_{name}.pth", "wb"))
            train_data = selector.transform(train_data)

        clf = Model(model_A, train_data, train_labels, name, inter_path, labels_loss)
        clf.Fit()

        labels_loss[name] = clf.get_class_weight()

    labels_loss[np.isnan(labels_loss)] = 0
    labels_loss[labels_loss < 0] = 0
    labels_loss.to_csv(f"{inter_path}/feature/labels_loss.csv", index=False)
    print(f"------------------------ 训练完成 ------------------------")

if __name__ == '__main__':

    inter_path = '../data/user_data'
    data_path = '../data/raw_data'
    feature_list = ['ember', 'section', 'imports', 'exports', 'words_1000', 'semantic', 'ember_section_ins_words', 'ember_section_ins_semantic']

    print(f"------------------------ 训练集特征工程 ------------------------")
    feature_engineering("train", data_path, inter_path)
    train_data_dict = load_data('train', feature_list, inter_path)

    train_lab_path = f"{inter_path}/train_y.npy"
    train_y = np.load(train_lab_path)

    train_model(train_data_dict, train_y, inter_path)

#!/usr/bin/env python # -*- encoding: utf-8 -*- @File    :   train.py @Contact :   <EMAIL> @License :   (C)Copyright 2021-2022 PowerLZY  @Modify Time      @Author               @Version    @Desciption ------------      -------               --------    ----------- 2021-10-04 15:03   PowerLZY&yuan_mes       1.0      保存训练模型，特征选择模型和单特征类别权重 # 使用了TF-IDF的特征做特征选择