'''
  Run a machine learning model over machine-learnable csvs, over each feature

Usage:
    csv_mlearner.py <i> [options]

Arguments:
    <i>                                The analysis JSON file generated by csv_detective
    --num_cores=<n> CORES                  Number of cores to use [default: 1:int]
'''

import numpy as np
import pandas as pd
from argopt import argopt
from joblib import delayed, Parallel
from sklearn.compose import ColumnTransformer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import f1_score
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from tqdm import tqdm

from src.data.find_ml_candidates import find_mlearnable_datasets

np.random.seed(0)


# for csv_id, csv_detective in categorical_continuous.items():
def mlearn_dataset(csv_id, csv_detective):
    resource_id = csv_id.split("/")[1]
    csv_path = f"/data/datagouv/csv_full/{resource_id}.csv"
    csv_encoding = csv_detective[csv_id]["encoding"]
    csv_sep = csv_detective[csv_id]["separator"]
    df = pd.read_csv(csv_path, encoding=csv_encoding, sep=csv_sep)

    categorical_features = csv_detective[csv_id]["categorical"]
    numerical_features = csv_detective[csv_id]["continous"]

    # We create the preprocessing pipelines for both numeric and categorical data.
    # numeric_features =
    numeric_transformer = Pipeline(steps=[
        ('imputer', SimpleImputer(strategy='median')),
        ('scaler', StandardScaler())])

    categorical_transformer = Pipeline(steps=[
        ('imputer', SimpleImputer(strategy='most_frequent')),
        ('onehot', OneHotEncoder(handle_unknown='ignore'))])

    other_transformer = Pipeline(steps=[
        # ('imputer', SimpleImputer(strategy='most_frequent')),
        ('vect', TfidfVectorizer(min_df=2, ngram_range=(1, 4)))])

    results_dict = {}
    for var in categorical_features:
        categorical_variables_copy = set(categorical_features)
        X = df.drop(var, axis=1)
        y = df[var]
        categorical_variables_copy.discard(var)
        categorical_features = list(categorical_variables_copy)
        other_features = np.setdiff1d(X.columns.values, categorical_features + numerical_features).tolist()

        preprocessor = ColumnTransformer(
            transformers=[
                ('num', numeric_transformer, numerical_features),
                ('cat', categorical_transformer, categorical_features),
                # ('other', other_transformer, other_features[0])
        ])

        # Append classifier to preprocessing pipeline.
        # Now we have a full prediction pipeline.
        clf = Pipeline(steps=[('preprocessor', preprocessor),
                              ('classifier', LogisticRegression())])

        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

        clf.fit(X_train, y_train)
        y_pred = clf.predict(X_test)
        fscore = f1_score(y_test, y_pred, average="macro")
        # tqdm.write(f"Predicted Class: {var}.\tModel f-score macro: {fscore}.\tDataset: {csv_id}")


###############################################################################
# Using the prediction pipeline in a grid search
###############################################################################
# Grid search can also be performed on the different preprocessing steps
# defined in the ``ColumnTransformer`` object, together with the classifier's
# hyperparameters as part of the ``Pipeline``.
# We will search for both the imputer strategy of the numeric preprocessing
# and the regularization parameter of the logistic regression using
# :class:`sklearn.model_selection.GridSearchCV`.


# param_grid = {
#     'preprocessor__num__imputer__strategy': ['mean', 'median'],
#     'classifier__C': [0.1, 1.0, 10, 100],
# }
#
# grid_search = GridSearchCV(clf, param_grid, cv=10)
# grid_search.fit(X_train, y_train)
#
# print(("best logistic regression from grid search: %.3f"
#        % grid_search.score(X_test, y_test)))


if __name__ == '__main__':
    parser = argopt(__doc__).parse_args()
    csv_detective_path = parser.i
    n_jobs = parser.num_cores

    categorical, continuous, categorical_continuous, csv_detective_json = find_mlearnable_datasets(csv_detective_path)
    # categorical_continuous = {"59591ca4a3a7291dcf9c8150/845bd585-7f17-4e88-a158-be3cd6526168":
    #                               categorical_continuous["59591ca4a3a7291dcf9c8150/845bd585-7f17-4e88-a158-be3cd6526168"]}
    if n_jobs < 2:
        job_output = []
        for id_dataset in tqdm(categorical_continuous):
            job_output.append(mlearn_dataset(id_dataset, csv_detective_json))
    else:
        job_output = Parallel(n_jobs=n_jobs)(
            delayed(mlearn_dataset)(id_dataset, csv_detective_json) for id_dataset in tqdm(categorical_continuous))


'''
  Run a machine learning model over machine-learnable csvs, over each feature

Usage:
    csv_mlearner.py <i> [options]

Arguments:
    <i>                                The analysis JSON file generated by csv_detective
    --num_cores=<n> CORES                  Number of cores to use [default: 1:int]
'''

import numpy as np
import pandas as pd
from argopt import argopt
from joblib import delayed, Parallel
from sklearn.compose import ColumnTransformer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import f1_score
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from tqdm import tqdm

from src.data.find_ml_candidates import find_mlearnable_datasets

np.random.seed(0)


# for csv_id, csv_detective in categorical_continuous.items():
def mlearn_dataset(csv_id, csv_detective):
    resource_id = csv_id.split("/")[1]
    csv_path = f"/data/datagouv/csv_full/{resource_id}.csv"
    csv_encoding = csv_detective[csv_id]["encoding"]
    csv_sep = csv_detective[csv_id]["separator"]
    df = pd.read_csv(csv_path, encoding=csv_encoding, sep=csv_sep)

    categorical_features = csv_detective[csv_id]["categorical"]
    numerical_features = csv_detective[csv_id]["continous"]

    # We create the preprocessing pipelines for both numeric and categorical data.
    # numeric_features =
    numeric_transformer = Pipeline(steps=[
        ('imputer', SimpleImputer(strategy='median')),
        ('scaler', StandardScaler())])

    categorical_transformer = Pipeline(steps=[
        ('imputer', SimpleImputer(strategy='most_frequent')),
        ('onehot', OneHotEncoder(handle_unknown='ignore'))])

    other_transformer = Pipeline(steps=[
        # ('imputer', SimpleImputer(strategy='most_frequent')),
        ('vect', TfidfVectorizer(min_df=2, ngram_range=(1, 4)))])

    results_dict = {}
    for var in categorical_features:
        categorical_variables_copy = set(categorical_features)
        X = df.drop(var, axis=1)
        y = df[var]
        categorical_variables_copy.discard(var)
        categorical_features = list(categorical_variables_copy)
        other_features = np.setdiff1d(X.columns.values, categorical_features + numerical_features).tolist()

        preprocessor = ColumnTransformer(
            transformers=[
                ('num', numeric_transformer, numerical_features),
                ('cat', categorical_transformer, categorical_features),
                # ('other', other_transformer, other_features[0])
        ])

        # Append classifier to preprocessing pipeline.
        # Now we have a full prediction pipeline.
        clf = Pipeline(steps=[('preprocessor', preprocessor),
                              ('classifier', LogisticRegression())])

        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

        clf.fit(X_train, y_train)
        y_pred = clf.predict(X_test)
        fscore = f1_score(y_test, y_pred, average="macro")
        # tqdm.write(f"Predicted Class: {var}.\tModel f-score macro: {fscore}.\tDataset: {csv_id}")


###############################################################################
# Using the prediction pipeline in a grid search
###############################################################################
# Grid search can also be performed on the different preprocessing steps
# defined in the ``ColumnTransformer`` object, together with the classifier's
# hyperparameters as part of the ``Pipeline``.
# We will search for both the imputer strategy of the numeric preprocessing
# and the regularization parameter of the logistic regression using
# :class:`sklearn.model_selection.GridSearchCV`.


# param_grid = {
#     'preprocessor__num__imputer__strategy': ['mean', 'median'],
#     'classifier__C': [0.1, 1.0, 10, 100],
# }
#
# grid_search = GridSearchCV(clf, param_grid, cv=10)
# grid_search.fit(X_train, y_train)
#
# print(("best logistic regression from grid search: %.3f"
#        % grid_search.score(X_test, y_test)))


if __name__ == '__main__':
    parser = argopt(__doc__).parse_args()
    csv_detective_path = parser.i
    n_jobs = parser.num_cores

    categorical, continuous, categorical_continuous, csv_detective_json = find_mlearnable_datasets(csv_detective_path)
    # categorical_continuous = {"59591ca4a3a7291dcf9c8150/845bd585-7f17-4e88-a158-be3cd6526168":
    #                               categorical_continuous["59591ca4a3a7291dcf9c8150/845bd585-7f17-4e88-a158-be3cd6526168"]}
    if n_jobs < 2:
        job_output = []
        for id_dataset in tqdm(categorical_continuous):
            job_output.append(mlearn_dataset(id_dataset, csv_detective_json))
    else:
        job_output = Parallel(n_jobs=n_jobs)(
            delayed(mlearn_dataset)(id_dataset, csv_detective_json) for id_dataset in tqdm(categorical_continuous))


Run a machine learning model over machine-learnable csvs, over each feature  Usage:     csv_mlearner.py <i> [options]  Arguments:     <i>                                The analysis JSON file generated by csv_detective     --num_cores=<n> CORES                  Number of cores to use [default: 1:int] # for csv_id, csv_detective in categorical_continuous.items(): # We create the preprocessing pipelines for both numeric and categorical data. # numeric_features = # ('imputer', SimpleImputer(strategy='most_frequent')), # ('other', other_transformer, other_features[0]) # Append classifier to preprocessing pipeline. # Now we have a full prediction pipeline. # tqdm.write(f"Predicted Class: {var}.\tModel f-score macro: {fscore}.\tDataset: {csv_id}") ############################################################################### # Using the prediction pipeline in a grid search ############################################################################### # Grid search can also be performed on the different preprocessing steps # defined in the ``ColumnTransformer`` object, together with the classifier's # hyperparameters as part of the ``Pipeline``. # We will search for both the imputer strategy of the numeric preprocessing # and the regularization parameter of the logistic regression using # :class:`sklearn.model_selection.GridSearchCV`. # param_grid = { #     'preprocessor__num__imputer__strategy': ['mean', 'median'], #     'classifier__C': [0.1, 1.0, 10, 100], # } # # grid_search = GridSearchCV(clf, param_grid, cv=10) # grid_search.fit(X_train, y_train) # # print(("best logistic regression from grid search: %.3f" #        % grid_search.score(X_test, y_test))) # categorical_continuous = {"59591ca4a3a7291dcf9c8150/845bd585-7f17-4e88-a158-be3cd6526168": #                               categorical_continuous["59591ca4a3a7291dcf9c8150/845bd585-7f17-4e88-a158-be3cd6526168"]}