<filename>core/feature_selection.py
# Author:           <NAME>
# Python Version:   3.6
# project:
# =============================================================================
"""this creates simple corelations of lags to a target variable. Uses person method"""

# =============================================================================
import numpy as np
import pandas as pd
from tqdm import tqdm


# =============================================================================

def create_lag_corelations_by_group(pdf, groups, target, lags=[1, 3, 4, 6, 12], limits=0.6, min_data_in_stream=12):
    """
    Get the person coefficiencts for lags of base features to the specified target.
    Alows the speification of filtering crietrias to limit the returned feeatures based on pearson coefficent values
    and amount of data in datastream.
    :param pdf: time series data. [pandas dataFrame]
    :param groups: unique groupings to create unique time stream [list of string]
    :param target: variable you want to corelated [string]
    :param lags: lags you want to explore [list of ints]
    :param limits: [float]
    :param min_data_in_stream: [int]

    :return: dataframe of pearson correaltions filtered by limits and min data
    """
    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
    pdf_lags_corr_target_list = []

    for lag in tqdm(lags):
        pdf_ = pdf.copy()

        for n, g in pdf_.groupby(groups):

            for col in g.select_dtypes(include=numerics):
                lag_column_name = '_'.join([col, 'LAG', str(lag) +'_'])
                g[lag_column_name] = g[col].shift(lag).values

            pdf_lags_cols = [col for col in g.columns if 'LAG_' + str(lag) in col] + [target]
            pdf_ = g[pdf_lags_cols]
            pdf_lags_corr = pdf_.corr().loc[:, target].reset_index()
            pdf_lags_corr.loc[:, 'LAG'] = lag
            pdf_lags_corr.loc[:, 'AMOUNT_OF_DATA'] = len(g)

            for name, group_attr in zip(n,groups):
              pdf_lags_corr.loc[:, group_attr] = name

            pdf_lags_corr_target_list.append(pdf_lags_corr)

    pdf_lags_corr_target = pd.concat(pdf_lags_corr_target_list).drop_duplicates()
    pdf_lags_corr_target = pdf_lags_corr_target[pdf_lags_corr_target["index"] != target]
    pdf_lags_corr_target.rename(columns={target: "CORELATION_TO_TARGET", "index": "FEATURE"}, inplace=True)

    if limits:
        pdf_lags_corr_target = pdf_lags_corr_target[
            (pdf_lags_corr_target["CORELATION_TO_TARGET"] > limits) | (
                    pdf_lags_corr_target["CORELATION_TO_TARGET"] < -1 * limits)]
    if min_data_in_stream:
        pdf_lags_corr_target = pdf_lags_corr_target[
            pdf_lags_corr_target["AMOUNT_OF_DATA"] > min_data_in_stream].reset_index(drop=True)

    return pdf_lags_corr_target

<filename>core/feature_selection.py
# Author:           <NAME>
# Python Version:   3.6
# project:
# =============================================================================
"""this creates simple corelations of lags to a target variable. Uses person method"""

# =============================================================================
import numpy as np
import pandas as pd
from tqdm import tqdm


# =============================================================================

def create_lag_corelations_by_group(pdf, groups, target, lags=[1, 3, 4, 6, 12], limits=0.6, min_data_in_stream=12):
    """
    Get the person coefficiencts for lags of base features to the specified target.
    Alows the speification of filtering crietrias to limit the returned feeatures based on pearson coefficent values
    and amount of data in datastream.
    :param pdf: time series data. [pandas dataFrame]
    :param groups: unique groupings to create unique time stream [list of string]
    :param target: variable you want to corelated [string]
    :param lags: lags you want to explore [list of ints]
    :param limits: [float]
    :param min_data_in_stream: [int]

    :return: dataframe of pearson correaltions filtered by limits and min data
    """
    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
    pdf_lags_corr_target_list = []

    for lag in tqdm(lags):
        pdf_ = pdf.copy()

        for n, g in pdf_.groupby(groups):

            for col in g.select_dtypes(include=numerics):
                lag_column_name = '_'.join([col, 'LAG', str(lag) +'_'])
                g[lag_column_name] = g[col].shift(lag).values

            pdf_lags_cols = [col for col in g.columns if 'LAG_' + str(lag) in col] + [target]
            pdf_ = g[pdf_lags_cols]
            pdf_lags_corr = pdf_.corr().loc[:, target].reset_index()
            pdf_lags_corr.loc[:, 'LAG'] = lag
            pdf_lags_corr.loc[:, 'AMOUNT_OF_DATA'] = len(g)

            for name, group_attr in zip(n,groups):
              pdf_lags_corr.loc[:, group_attr] = name

            pdf_lags_corr_target_list.append(pdf_lags_corr)

    pdf_lags_corr_target = pd.concat(pdf_lags_corr_target_list).drop_duplicates()
    pdf_lags_corr_target = pdf_lags_corr_target[pdf_lags_corr_target["index"] != target]
    pdf_lags_corr_target.rename(columns={target: "CORELATION_TO_TARGET", "index": "FEATURE"}, inplace=True)

    if limits:
        pdf_lags_corr_target = pdf_lags_corr_target[
            (pdf_lags_corr_target["CORELATION_TO_TARGET"] > limits) | (
                    pdf_lags_corr_target["CORELATION_TO_TARGET"] < -1 * limits)]
    if min_data_in_stream:
        pdf_lags_corr_target = pdf_lags_corr_target[
            pdf_lags_corr_target["AMOUNT_OF_DATA"] > min_data_in_stream].reset_index(drop=True)

    return pdf_lags_corr_target

# Author:           <NAME> # Python Version:   3.6 # project: # ============================================================================= this creates simple corelations of lags to a target variable. Uses person method # ============================================================================= # ============================================================================= Get the person coefficiencts for lags of base features to the specified target.     Alows the speification of filtering crietrias to limit the returned feeatures based on pearson coefficent values     and amount of data in datastream.     :param pdf: time series data. [pandas dataFrame]     :param groups: unique groupings to create unique time stream [list of string]     :param target: variable you want to corelated [string]     :param lags: lags you want to explore [list of ints]     :param limits: [float]     :param min_data_in_stream: [int]      :return: dataframe of pearson correaltions filtered by limits and min data