from typing import Dict, List, Union

import pandas as pd

from toucan_data_sdk.utils.helpers import (
    check_params_columns_duplicate,
    ParamsValueError
)

def add_missing_row(
    df: pd.DataFrame,
    id_cols: List[str],
    reference_col: str,
    complete_index: Union[Dict[str, str], List[str]] = None,
    method: str = None,
    cols_to_keep: List[str] = None
) -> pd.DataFrame:
    """
    Add missing row to a df base on a reference column

    ---

    ### Parameters

    *mandatory :*
    - `id_cols` (*list of str*): names of the columns used to create each group
    - `reference_col` (*str*): name of the column used to identify missing rows

    *optional :*
    - `complete_index` (*list* or *dict*): [A, B, C] a list of values used to add missing rows.
      It can also be a dict to declare a date range.
      By default, use all values of reference_col.
    - `method` (*str*): by default all missing rows are added. The possible values are :
        - `"between"` : add missing rows having their value between min and max values for each group,
        - `"between_and_after"` : add missing rows having their value bigger than min value for each group.
        - `"between_and_before"` : add missing rows having their value smaller than max values for each group.
    - `cols_to_keep` (*list of str*): name of other columns to keep, linked to the reference_col.

    ---

    ### Example

    **Input**

    YEAR | MONTH | NAME
    :---:|:---:|:--:
    2017|1|A
    2017|2|A
    2017|3|A
    2017|1|B
    2017|3|B

    ```cson
    add_missing_row:
      id_cols: ['NAME']
      reference_col: 'MONTH'
    ```

    **Output**

    YEAR | MONTH | NAME
    :---:|:---:|:--:
    2017|1|A
    2017|2|A
    2017|3|A
    2017|1|B
    2017|2|B
    2017|3|B

    """
    if cols_to_keep is None:
        cols_for_index = [reference_col]
    else:
        cols_for_index = [reference_col] + cols_to_keep
    check_params_columns_duplicate(id_cols + cols_for_index)

    if method == 'between' or method == 'between_and_after':
        df['start'] = df.groupby(id_cols)[reference_col].transform(min)
        id_cols += ['start']
    if method == 'between' or method == 'between_and_before':
        df['end'] = df.groupby(id_cols)[reference_col].transform(max)
        id_cols += ['end']

    names = id_cols + cols_for_index
    new_df = df.set_index(names)
    index_values = df.groupby(id_cols).sum().index.values

    if complete_index is None:
        complete_index = df.groupby(cols_for_index).sum().index.values
    elif isinstance(complete_index, dict):
        if complete_index['type'] == 'date':
            freq = complete_index['freq']
            date_format = complete_index['format']
            start = complete_index['start']
            end = complete_index['end']
            if isinstance(freq, dict):
                freq = pd.DateOffset(**{k: int(v) for k, v in freq.items()})
            complete_index = pd.date_range(start=start, end=end, freq=freq)
            complete_index = complete_index.strftime(date_format)
        else:
            raise ParamsValueError(f'Unknown complete index type: '
                                   f'{complete_index["type"]}')

    if not isinstance(index_values[0], tuple):
        index_values = [(x,) for x in index_values]
    if not isinstance(complete_index[0], tuple):
        complete_index = [(x,) for x in complete_index]
    new_tuples_index = [x + y for x in index_values for y in complete_index]

    new_index = pd.MultiIndex.from_tuples(new_tuples_index, names=names)
    new_df = new_df.reindex(new_index).reset_index()

    if method == 'between' or method == 'between_and_after':
        new_df = new_df[new_df[reference_col] >= new_df['start']]
        del new_df['start']
    if method == 'between' or method == 'between_and_before':
        new_df = new_df[new_df[reference_col] <= new_df['end']]
        del new_df['end']

    return new_df


import logging
import os
import shutil
import tempfile
import zipfile
import pandas as pd
import joblib
from toucan_client import ToucanClient
from .utils.helpers import slugify
from .utils.traceback import load_traceback

logger = logging.getLogger(__name__)

class ToucanDataSdk:
    def __init__(self, instance_url, auth, small_app=None, stage="staging"):
        instance_url = instance_url.strip().rstrip('/')
        if small_app is None:
            small_app = instance_url.split('/')[-1]
            instance_url = '/'.join(instance_url.split('/')[:-1])
        self.small_app_url = instance_url + (('/' + small_app) if small_app else '')
        self.client = ToucanClient(self.small_app_url, auth=auth, stage=stage)
        self.EXTRACTION_CACHE_PATH = os.path.join(
            'extraction_cache',
            slugify(instance_url, separator='_'),
            small_app
        )

    def get_datasources(self, domains=None):
        if domains is not None and isinstance(domains, list):
            dfs = {}
            domains_cache = [domain for domain in domains
                             if self.cache_exists(domain)]
            domains_sdk = list(set(domains) - set(domains_cache))

            if len(domains_cache) > 0:
                dfs.update(self.read_from_cache(domains_cache))
            if len(domains_sdk) > 0:
                dfs.update(self.read_datasources_from_sdk(domains_sdk))
        else:
            if self.cache_exists():
                dfs = self.read_from_cache()
            else:
                dfs = self.read_datasources_from_sdk()
        return dfs

    # alias
    get_dfs = get_datasources

    def get_domains(self, domains=None):
        if domains is None:
            domains = [meta["domain"] for meta in self.client.metadata.get().json()]
        domains_cache = [domain for domain in domains if self.cache_exists(domain)]
        domains_sdk = [domain for domain in domains if domain not in domains_cache]
        if len(domains_cache) > 0:
            dfs = self.read_from_cache(domains_cache)
        if len(domains_sdk) > 0:
            dfs = self.read_domains_from_sdk(domains_sdk)
        return dfs

    def invalidate_cache(self, domains=None):
        if domains is not None and isinstance(domains, list):
            for domain in domains:
                try:
                    os.remove(os.path.join(self.EXTRACTION_CACHE_PATH, domain))
                except OSError:
                    pass
        else:
            try:
                shutil.rmtree(self.EXTRACTION_CACHE_PATH)
            except (OSError, IOError) as e:  # For Python 2.7+ compatibility
                logger.error('failed to remove cache for : ' + str(e))

    def get_augment(self):
        return self.client.config.augment.get().text

    def get_etl(self):
        return self.client.config.etl.get().json()

    def query_basemaps(self, query):
        if isinstance(query, dict):
            return self.client.basemaps.post(json=query).json()
        else:
            raise InvalidQueryError(f'Query {query} should be a dict, {type(query)} found.')

    def read_datasources_from_sdk(self, domains=None):
        # Extract all domains if domains_sdk is null
        resp = self.client.sdk.post(json={'domains': domains})
        resp.raise_for_status()
        dfs = extract(resp.content)
        self.write(dfs)
        logger.info(f'Data {domains} fetched and cached')
        return dfs

    def read_domains_from_sdk(self, domains):
        dfs = {}
        for domain in domains:
            # first page
            data = self.client.output_domain[domain].post().json()
            rows = data['result']
            last_doc_id = data['lastDocId']

            # next pages
            while last_doc_id:
                data = self.client.output_domain[domain][last_doc_id].post().json()
                rows += data['result']
                last_doc_id = data['lastDocId']
            df = pd.DataFrame.from_dict(rows).drop(columns='_id')
            dfs[domain] = df
        self.write(dfs)
        return dfs

    def read_from_cache(self, domains=None):
        """
        Returns:
            dict: Dict[str, DataFrame]
        """
        logger.info(f'Reading data from cache ({self.EXTRACTION_CACHE_PATH})')
        if domains is not None and isinstance(domains, list):
            dfs = {domain: self.read_entry(domain) for domain in domains}
        else:
            dfs = {name: self.read_entry(name)
                   for name in os.listdir(self.EXTRACTION_CACHE_PATH)}
        return dfs

    def read_entry(self, file_name):
        """
        Args:
            file_name (str):

        Returns:
            pd.DataFrame:
        """
        file_path = os.path.join(self.EXTRACTION_CACHE_PATH, file_name)
        logger.info(f'Reading cache entry: {file_path}')
        return joblib.load(file_path)

    def write(self, dfs):
        """
        Args:
            data (str | byte):

        Returns:
            dict: Dict[str, DataFrame]
        """
        if not os.path.exists(self.EXTRACTION_CACHE_PATH):
            os.makedirs(self.EXTRACTION_CACHE_PATH)

        for name, df in dfs.items():
            file_path = os.path.join(self.EXTRACTION_CACHE_PATH, name)
            joblib.dump(df, filename=file_path)
            logger.info(f'Cache entry added: {file_path}')

    def cache_exists(self, domain=None):
        if domain is not None:
            path = os.path.join(self.EXTRACTION_CACHE_PATH, domain)
            return os.path.exists(path) and os.path.isfile(path)
        else:
            path = self.EXTRACTION_CACHE_PATH
            return os.path.exists(path) and os.path.isdir(path)

    def load_latest_traceback(self):
        tb = self.client.tracebacks.latest.get()
        if tb.ok is False:
            return tb.json()
        else:
            with open('.tb.dump', 'wb') as f:
                f.write(tb.content)
            return load_traceback('.tb.dump')

def extract_zip(zip_file_path):
    """
    Returns:
        dict: Dict[str, DataFrame]
    """
    dfs = {}
    with zipfile.ZipFile(zip_file_path, mode='r') as z_file:
        names = z_file.namelist()
        for name in names:
            content = z_file.read(name)
            _, tmp_file_path = tempfile.mkstemp()
            try:
                with open(tmp_file_path, 'wb') as tmp_file:
                    tmp_file.write(content)

                dfs[name] = joblib.load(tmp_file_path)
            finally:
                shutil.rmtree(tmp_file_path, ignore_errors=True)
    return dfs

def extract(data):
    """
    Args:
        data (str | byte):

    Returns:
        dict: Dict[str, DataFrame]

    """
    _, tmp_file_path = tempfile.mkstemp()
    try:
        with open(tmp_file_path, 'wb') as tmp_file:
            tmp_file.write(data)

        if zipfile.is_zipfile(tmp_file_path):
            return extract_zip(tmp_file_path)
        else:
            raise DataSdkError('Unsupported file type')
    finally:
        shutil.rmtree(tmp_file_path, ignore_errors=True)

class DataSdkError(Exception):
    pass

class InvalidQueryError(Exception):
    pass


from toucan_data_sdk.utils.helpers import slugify

def get_category_cols(df, threshold):
    obj_df = df.select_dtypes(include=['object'])
    return [col for col in obj_df.columns if len(obj_df[col].unique()) < threshold]

def get_int_cols(df):
    float_df = df.select_dtypes(include=['floating'])
    return [col for col in float_df.columns if all(x.is_integer() for x in float_df[col])]

def clean_dataframe(df, is_slugify=True, threshold=50, rename_cols=None):
    """
    This method is used to:
    - slugify the column names (if slugify is set to True)
    - convert columns to 'category' (if len(unique) < threshold) or 'int'
    - clean the dataframe and rename if necessary
    """
    if is_slugify:
        df = df.rename(columns=slugify)

    df = df.dropna(axis=1, how='all')
    for column in get_category_cols(df, threshold=threshold):
        df[column] = df[column].astype('category')
    for column in get_int_cols(df):
        df[column] = df[column].astype(int)

    if rename_cols is not None:
        df = df.rename(columns=rename_cols)

    return df


from typing import List

from toucan_data_sdk.utils.helpers import check_params_columns_duplicate

def compute_ffill_by_group(
        df,
        id_cols: List[str],
        reference_cols: List[str],
        value_col: str
):
    """
    Compute `ffill` with `groupby`
    Dedicated method as there is a performance issue with a simple groupby/fillna (2017/07)
    The method `ffill` propagates last valid value forward to next values.

    ---

    ### Parameters

    *mandatory :*
    - `id_cols` (*list of str*): names of columns used to create each group.
    - `reference_cols` (*list of str*): names of columns used to sort.
    - `value_col` (*str*): name of the columns to fill.

    ---

    ### Example

    **Input**

    name | rank | value
    :------:|:--------------:|:--------:
    A | 1 | 2
    A | 2 | 5
    A | 3 | null
    B | 1 | null
    B | 2 | 7

    ```cson
    compute_ffill_by_group:
      id_cols: ['name']
      reference_cols: ['rank']
      value_col: 'value'
    ```

    **Ouput**

    name | rank | value
    :------:|:--------------:|:--------:
    A | 1 | 2
    A | 2 | 5
    A | 3 | 5
    B | 1 | null
    B | 2 | 7
    """
    check_params_columns_duplicate(id_cols + reference_cols + [value_col])
    df = df.sort_values(by=id_cols + reference_cols)
    df = df.set_index(id_cols)
    df['fill'] = 1 - df[value_col].isnull().astype(int)
    df['fill'] = df.groupby(
        level=list(range(0, len(id_cols) - 1))
    )['fill'].cumsum()
    df[value_col] = df[value_col].ffill()
    df.loc[df['fill'] == 0, value_col] = None
    del df['fill']
    return df.reset_index()


from itertools import product
from typing import List

import numpy as np
import pandas as pd

def fake_data_generator(conf: List[dict]) -> pd.DataFrame:
    """
    `conf` is a list of dictionaries like
      {'type': 'label', 'values': ['Paris', 'Marseille', 'Lyons'], 'name': 'Cities'}
    and each dictionary will add a column.

    There are two different behaviours:
    - type: 'label' -> the new column will be taken into account for a cartesian product
                       with all other labels
    - type: 'number' -> the new column will contain simple numbers
    """

    # First create all the lines with the cartesian product of all the
    # possible values of 'label' columns
    label_confs = [x for x in conf if x['type'] == 'label']
    label_names = [x['name'] for x in label_confs]
    label_values = [x['values'] for x in label_confs]
    df = pd.DataFrame(list(product(*label_values)), columns=label_names)

    # Then add all the 'number' columns
    number_confs = [x for x in conf if x['type'] == 'number']
    for num_conf in number_confs:
        num_column = np.random.uniform(low=num_conf['min'], high=num_conf['max'], size=df.shape[0])
        df[num_conf['name']] = num_column.round(num_conf.get('digits', 4))

    return df


import pandas as pd
from typing import List

def two_values_melt(
    df,
    first_value_vars: List[str],
    second_value_vars: List[str],
    var_name: str,
    value_name: str
):
    """
    Transforms one or multiple columns into rows.
    Unlike melt function, two value columns can be returned by
    the function (e.g. an evolution column and a price column)

    ---

    ### Parameters

    *mandatory :*
    - `first_value_vars` (*list of str*): name of the columns corresponding to the first returned value column
    - `second_value_vars` (*list of str*): name of the columns corresponding to the second returned value column
    - `var_name` (*str*): name of the column containing values in first_value_vars
    - `value_name` (*str*): suffix of the two value columns (suffix_first / suffix_second)
    ---

    ### Example

    **Input**

    |    Region |      avg |       total |  evo_avg |   evo_total |
    |:---------:|:--------:|:-----------:|:--------:|:-----------:|
    |         A |        50|         100 |        1 |           4 |
    |         B |       40 |         250 |        2 |           5 |

    ```cson
    two_values_melt:
      first_value_vars: ["avg", "total"]
      second_value_vars: ["evo_avg", "evo_total"]
      var_name: "type"
      value_name: "value"
    ```

    **Output**

    |    Region |     type |  value_first |  value_second |
    |:---------:|:--------:|:------------:|:-------------:|
    |         A |       avg|           50 |             1 |
    |         A |     total|          100 |             4 |
    |         B |       avg|           40 |             2 |
    |         B |       avg|          250 |             5 |
    """
    value_name_first = value_name + '_first'
    value_name_second = value_name + '_second'

    # Melt on the first value columns
    melt_first_value = pd.melt(df,
                               id_vars=[col for col in list(df) if
                                        col not in first_value_vars],
                               value_vars=first_value_vars,
                               var_name=var_name,
                               value_name=value_name_first)
    melt_first_value.drop(second_value_vars, axis=1, inplace=True)

    # Melt on the second value columns
    melt_second_value = pd.melt(df,
                                id_vars=[col for col in list(df) if
                                         col not in second_value_vars],
                                value_vars=second_value_vars,
                                var_name=var_name,
                                value_name=value_name_second)

    # Since there are two value columns, there is no need to keep the
    # second_value_vars names. And it will make things easier for the merge.
    normalize_types = {k: v for k, v in zip(second_value_vars, first_value_vars)}
    melt_second_value.replace(normalize_types, inplace=True)
    melt_second_value.drop(first_value_vars, axis=1, inplace=True)

    on_cols = list(melt_first_value)
    on_cols.remove(value_name_first)
    return pd.merge(melt_first_value, melt_second_value, on=on_cols, how='outer')


from typing import List, Any

import numpy as np

__all__ = (
    'lower',
    'upper',
    'title',
    'capitalize',
    'swapcase',
    'length',
    'isalnum',
    'isalpha',
    'isdigit',
    'isspace',
    'islower',
    'isupper',
    'istitle',
    'isnumeric',
    'isdecimal',
    'strip',
    'lstrip',
    'rstrip',
    'center',
    'ljust',
    'rjust',
    'split',
    'rsplit',
    'partition',
    'rpartition',
    'find',
    'rfind',
    'index',
    'rindex',
    'startswith',
    'endswith',
    'concat',
    'contains',
    'repeat',
    'replace_pattern',
    # 'slice',
    # 'slice_replace',
    # 'count'
)

###################################################################################################
#                              METHODS WITH NO EXTRA PARAMETERS
#
# All these functions have the same signature:
# :param df: the dataframe
# :param column: the column
# :param new_column: the destination column (if not set, `column` will be used)
# :return: the transformed dataframe
###################################################################################################

def _generate_basic_str_postprocess(method_name, docstring):
    def f(df, column: str, new_column: str = None):
        method = getattr(df[column].str, method_name)
        new_column = new_column or column
        df.loc[:, new_column] = method()
        return df

    f.__name__ = method_name
    f.__doc__ = f'''
    {docstring}
    See [pandas doc](
    https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.str.{method_name}.html) for more information

    ---

    ### Parameters

    *mandatory :*
    - `column` (*str*): the column

    *optional :*
    - `new_column` (*str*): the destination column (if not set, `column` will be used)
    '''
    return f

doc = 'Compute length of each string of `column`'
length = _generate_basic_str_postprocess('len', doc)

# lower, upper, capitalize, title, swapcase
###################################################################################################
doc = 'Converts all characters of `column` to lowercase.'
lower = _generate_basic_str_postprocess('lower', doc)

doc = 'Converts all characters of `column` to uppercase.'
upper = _generate_basic_str_postprocess('upper', doc)

doc = 'Converts first character to uppercase and remaining ' \
      'to lowercase for each line of `column`.'
capitalize = _generate_basic_str_postprocess('capitalize', doc)

doc = 'Converts first character to uppercase and remaining ' \
      'to lowercase for each word of each line of `column`.'
title = _generate_basic_str_postprocess('title', doc)

doc = 'Converts uppercase to lowercase and lowercase to uppercase for each word of `column`.'
swapcase = _generate_basic_str_postprocess('swapcase', doc)

# isalnum, isalpha, isdigit, isspace, islower, isupper, istitle, isnumeric, isdecimal
###################################################################################################
doc = 'Check whether all characters in each string in `column` are alphanumeric'
isalnum = _generate_basic_str_postprocess('isalnum', doc)

doc = 'Check whether all characters in each string in `column` are alphabetic'
isalpha = _generate_basic_str_postprocess('isalpha', doc)

doc = 'Check whether all characters in each string in `column` are digits'
isdigit = _generate_basic_str_postprocess('isdigit', doc)

doc = 'Check whether all characters in each string in `column` are whitespace'
isspace = _generate_basic_str_postprocess('isspace', doc)

doc = 'Check whether all characters in each string in `column` are lowercase'
islower = _generate_basic_str_postprocess('islower', doc)

doc = 'Check whether all characters in each string in `column` are uppercase'
isupper = _generate_basic_str_postprocess('isupper', doc)

doc = 'Check whether all characters in each string in `column` are titlecase'
istitle = _generate_basic_str_postprocess('istitle', doc)

doc = 'Check whether all characters in each string in `column` are numeric'
isnumeric = _generate_basic_str_postprocess('isnumeric', doc)

doc = 'Check whether all characters in each string in `column` are decimal'
isdecimal = _generate_basic_str_postprocess('isdecimal', doc)

###################################################################################################
#                                        STRIP METHODS
#
# All these functions have the same signature:
# :param df: the dataframe
# :param column: the column
# :param to_strip: (str: None) set of characters to be removed
# :param new_column: the destination column (if not set, `column` will be used)
# :return: the transformed dataframe
###################################################################################################
def _generate_strip_str_postprocess(method_name, docstring):
    def f(df, column: str, *, to_strip: str = None, new_column: str = None):
        method = getattr(df[column].str, method_name)
        new_column = new_column or column
        df.loc[:, new_column] = method(to_strip)
        return df

    f.__name__ = method_name
    f.__doc__ = f'''
    {docstring}
    See [pandas doc](
    https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.str.{method_name}.html) for more information

    ---

    ### Parameters

    *mandatory :*
    - `column` (*str*): the column

    *optional :*
    - `to_strip` (*str*): set of characters to be removed
    - `new_column` (*str*): the destination column (if not set, `column` will be used)
    '''
    return f

doc = 'Strip whitespace (including newlines) from each string in `column` from both sides'
strip = _generate_strip_str_postprocess('strip', doc)

doc = 'Strip whitespace (including newlines) from each string in `column` from left side'
lstrip = _generate_strip_str_postprocess('lstrip', doc)

doc = 'Strip whitespace (including newlines) from each string in `column` from left side'
rstrip = _generate_strip_str_postprocess('rstrip', doc)

###################################################################################################
#                              METHODS with `width` and `fillchar`
#
# All these functions have the same signature:
# :param df: the dataframe
# :param column: the column
# :param width: (int) minimum width
# :param fillchar: (default: \' \') additional character for filling
# :param new_column: the destination column (if not set, `column` will be used)
# :return: the transformed dataframe
###################################################################################################

def _generate_width_str_postprocess(method_name, docstring):
    def f(df, column: str, *, width: int, fillchar: str = ' ', new_column: str = None):
        method = getattr(df[column].str, method_name)
        new_column = new_column or column
        df.loc[:, new_column] = method(width, fillchar=fillchar)
        return df

    f.__name__ = method_name
    f.__doc__ = f'''
    {docstring}
    See [pandas doc](
    https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.str.{method_name}.html) for more information

    ---

    ### Parameters

    *mandatory :*
    - `column` (*str*): the column
    - `width` (*int*): minimum widt

    *optional :*
    - `fillchar` (*str*): additional character for filling
    - `new_column` (*str*): the destination column (if not set, `column` will be used)
    '''
    return f

doc = 'Filling left and right side of strings in `column` with an additional character'
center = _generate_width_str_postprocess('center', doc)

doc = 'Filling right side of strings in `column` with an additional character'
ljust = _generate_width_str_postprocess('ljust', doc)

doc = 'Filling left side of strings in `column` with an additional character'
rjust = _generate_width_str_postprocess('rjust', doc)

###################################################################################################
#                                        SPLIT METHODS
#
# All these functions have the same signature:
# :param df: the dataframe
# :param column: the column
# :param new_columns: the destination columns
#        (if not set, columns `column_1`, ..., `column_n` will be created)
# :param sep: (default: \' \') string or regular expression to split on
# :param limit: (default: None) limit number of splits in output
# :return: the transformed dataframe
###################################################################################################
def _generate_split_str_postprocess(method_name, docstring):
    def f(df, column: str, *, new_columns: List[str] = None, sep: str = ' ', limit: int = None):
        method = getattr(df[column].str, method_name)
        df_split = method(pat=sep, n=limit, expand=True)
        nb_cols = df_split.shape[1]
        if new_columns and (not isinstance(new_columns, list) or nb_cols > len(new_columns)):
            raise ValueError(f"'new_columns' should be a list with at least {nb_cols} elements")
        if new_columns is None:
            new_columns = [f'{column}_{i}' for i in range(1, nb_cols + 1)]
        df[new_columns[:nb_cols]] = df_split
        return df

    f.__name__ = method_name
    f.__doc__ = f'''
    {docstring}
    See [pandas doc](
    https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.str.{method_name}.html) for more information

    ---

    ### Parameters

    *mandatory :*
    - `column` (*str*): the column

    *optional :*
    - `sep` (*str*): string or regular expression to split on
    - `limit` (*int*): limit number of splits in output (by default, there is no limit)
    - `new_columns` (*list*): the destination columns (by default, new columns will be added automatically)
    '''
    return f

doc = 'Split each string in the caller’s values by given pattern, propagating NaN values'
split = _generate_split_str_postprocess('split', doc)

doc = 'Split each string `column` by the given delimiter string, ' \
      'starting at the end of the string and working to the front'
rsplit = _generate_split_str_postprocess('rsplit', doc)

###################################################################################################
#                                        PARTITION METHODS
#
# All these functions have the same signature:
# :param df: the dataframe
# :param column: the column
# :param new_columns: the 3 destination columns
# :param sep: (default: \' \') string or regular expression to split on
# :return: the transformed dataframe
###################################################################################################
def _generate_partition_str_postprocess(method_name, docstring):
    def f(df, column: str, *, new_columns: List[str], sep: str = ' '):
        if len(new_columns) != 3:
            raise ValueError('`new_columns` must have 3 columns exactly')
        method = getattr(df[column].str, method_name)
        df[new_columns] = method(sep)
        return df

    f.__name__ = method_name
    f.__doc__ = f'''
    {docstring}
    See [pandas doc](
    https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.str.{method_name}.html) for more information

    ---

    ### Parameters

    *mandatory :*
    - `column` (*str*): the column
    - `new_columns` (*list*): the 3 destination columns

    *optional :*
    - `sep` (*str*): string or regular expression to split on
    '''
    return f

doc = 'Split the string at the first occurrence of sep, and return 3 elements containing ' \
      'the part before the separator, the separator itself, and the part after the separator. ' \
      'If the separator is not found, return 3 elements containing the string itself, ' \
      'followed by two empty strings.'
partition = _generate_partition_str_postprocess('partition', doc)

doc = 'Split the string at the last occurrence of sep, and return 3 elements containing ' \
      'the part before the separator, the separator itself, and the part after the separator. ' \
      'If the separator is not found, return 3 elements containing two empty strings, ' \
      'followed by the string itself.'
rpartition = _generate_partition_str_postprocess('rpartition', doc)

###################################################################################################
#                                   INDEX AND FIND METHODS
#
# All these functions have the same signature:
# :param df: the dataframe
# :param column: the column
# :param new_column: the destination column (if not set, `column` will be used)
# :param sub: substring being searched
# :param start: (default: 0) left edge index
# :param end: (default: None) right edge index
# :return: the transformed dataframe
###################################################################################################
def _generate_find_str_postprocess(method_name, docstring):
    def f(df, column: str, *, sub: str, start: int = 0, end: int = None, new_column: str = None):
        method = getattr(df[column].str, method_name)
        new_column = new_column or column
        df.loc[:, new_column] = method(sub, start, end)
        return df

    f.__name__ = method_name
    f.__doc__ = f'''
    {docstring}
    See [pandas doc](
    https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.str.{method_name}.html) for more information

    ---

    ### Parameters

    *mandatory :*
    - `column` (*str*): the column
    - `sub` (*str*): substring being searched

    *optional :*
    - `start` (*int*): left edge index
    - `end` (*int*): right edge index
    - `new_column` (*str*): the destination column (if not set, `column` will be used)
    '''
    return f

doc = 'Return lowest indexes in each strings in `column` where the substring ' \
      'is fully contained between [start:end]. Return -1 on failure'
find = _generate_find_str_postprocess('find', doc)

doc = 'Return highest indexes in each strings in `column` where the substring ' \
      'is fully contained between [start:end]. Return -1 on failure'
rfind = _generate_find_str_postprocess('rfind', doc)

doc = 'Return lowest indexes in each strings where the substring is fully contained ' \
      'between [start:end]. This is the same as str.find except instead of returning -1, ' \
      'it raises a ValueError when the substring is not found'
index = _generate_find_str_postprocess('index', doc)

doc = 'Return highest indexes in each strings where the substring is fully contained ' \
      'between [start:end]. This is the same as str.find except instead of returning -1, ' \
      'it raises a ValueError when the substring is not found'
rindex = _generate_find_str_postprocess('rindex', doc)

###################################################################################################
#                                  STARTSWITH/ENDSWITH METHODS
#
# All these functions have the same signature:
# :param df: the dataframe
# :param column: the column
# :param new_column: the destination column (if not set, `column` will be used)
# :param pat: character sequence
# :param na: (default: NaN) object shown if element tested is not a string
# :return: the transformed dataframe
###################################################################################################
def _generate_with_str_postprocess(method_name, docstring):
    def f(df, column: str, *, pat: str, na: Any = np.nan, new_column: str = None):
        method = getattr(df[column].str, method_name)
        new_column = new_column or column
        df.loc[:, new_column] = method(pat, na=na)
        return df

    f.__name__ = method_name
    f.__doc__ = f'{docstring}\n' \
                f':param df: the dataframe\n' \
                f':param column: the column\n' \
                f':param new_column: the destination column (if not set, `column` will be used)\n' \
                f':param pat: character sequence\n' \
                f':param na: (default: NaN) object shown if element tested is not a string\n' \
                f':return: the transformed dataframe'
    f.__doc__ = f'''
    {docstring}
    See [pandas doc](
    https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.str.{method_name}.html) for more information

    ---

    ### Parameters

    *mandatory :*
    - `column` (*str*): the column
    - `pat` (*str*): character sequence

    *optional :*
    - `na`: object shown if element tested is not a string
    - `new_column` (*str*): the destination column (if not set, `column` will be used)
    '''
    return f

doc = 'Test if the start of each string element matches a pattern.'
startswith = _generate_with_str_postprocess('startswith', doc)

doc = 'Test if the end of each string element matches a pattern.'
endswith = _generate_with_str_postprocess('endswith', doc)

###################################################################################################
#                                        OTHER METHODS
###################################################################################################
def concat(
        df,
        *,
        columns: List[str],
        new_column: str,
        sep: str = None
):
    """
    Concatenate `columns` element-wise
    See [pandas doc](
    https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.str.cat.html) for more information

    ---

    ### Parameters

    *mandatory :*
    - `columns` (*list*): list of columns to concatenate (at least 2 columns)
    - `new_column` (*str*): the destination column

    *optional :*
    - `sep` (*str*): the separator
    """
    if len(columns) < 2:
        raise ValueError('The `columns` parameter needs to have at least 2 columns')
    first_col, *other_cols = columns
    df.loc[:, new_column] = df[first_col].astype(str).str.cat(df[other_cols].astype(str), sep=sep)
    return df

def contains(
        df,
        column: str,
        *,
        pat: str,
        new_column: str = None,
        case: bool = True,
        na: Any = None,
        regex: bool = True
):
    """
    Test if pattern or regex is contained within strings of `column`
    See [pandas doc](
    https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.str.contains.html) for more information

    ---

    ### Parameters

    *mandatory :*
    - `column` (*str*): the column
    - `pat` (*str*): character sequence or regular expression.

    *optional :*
    - `new_column` (*str*): the destination column (if not set, `column` will be used)
    - `case` (*boolean*): if true, case sensitive.
    - `na`: fill value for missing values.
    - `regex` (*boolean*): default true
    """
    new_column = new_column or column
    df.loc[:, new_column] = df[column].str.contains(pat, case=case, na=na, regex=regex)
    return df

def repeat(
        df,
        column: str,
        *,
        times: int,
        new_column: str = None
):
    """
    Duplicate each string in `column` by indicated number of time
    See [pandas doc](
    https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.str.repeat.html) for more information

    ---

    ### Parameters

    *mandatory :*
    - `column` (*str*): the column
    - `times` (*int*): times to repeat the string

    *optional :*
    - `new_column` (*str*): the destination column (if not set, `column` will be used)
    """
    new_column = new_column or column
    df.loc[:, new_column] = df[column].str.repeat(times)
    return df

def replace_pattern(
        df,
        column: str,
        *,
        pat: str,
        repl: str,
        new_column: str = None,
        case: bool = True,
        regex: bool = True
):
    """
    Replace occurrences of pattern/regex in `column` with some other string
    See [pandas doc](
    https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.str.replace.html) for more information

    ---

    ### Parameters

    *mandatory :*
    - `column` (*str*): the column
    - `pat` (*str*): character sequence or regular expression
    - `repl` (*str*): replacement string

    *optional :*
    - `new_column` (*str*): the destination column (if not set, `column` will be used)
    - `case` (*boolean*): if true, case sensitive.
    - `regex` (*boolean*): default true
    """
    new_column = new_column or column
    df.loc[:, new_column] = df[column].str.replace(pat, repl, case=case, regex=regex)
    return df


"""
Implements a series of decorators functions with decorator arguments (logger).
Decorators modify the behaviour of a function to execute code before and after
the original function.

They are a very useful pattern for logging, timing functions, retrying execution,
checking credentials, patching methods or caching for example.

Examples:

    # augment.py
    import logging
    from toucan_data_sdk.utils.decorators import log_message, log_time, log_shapes

    logger = logging.getLogger(__name__)

    @log_shapes(logger)
    @log_message(logger, "Replace affaires with parent affaires")
    @log_time(logger)
    def parse_reseau(df):
        # Do transformations
        return df

Output:
    >>> 2016-06-23 11:21 - INFO - small_apps.demo.preprocess.augment
                                | parse_reseau - time: 50.1 ms
    >>> 2016-06-23 11:21 - INFO - small_apps.demo.preprocess.augment
                                | parse_reseau - Replace affaires with parent affaires
    >>> 2016-06-23 11:21 - INFO - small_apps.demo.preprocess.augment
                                | parse_reseau - [(5821, 210)] -> [(662, 15)]

Note:
    You can apply multiple decorators to the same function.
    The decorators get applied in order from bottom to top.

"""
import collections
import logging
import time
from functools import wraps, partial
from hashlib import md5
from threading import current_thread

import joblib
import pandas as pd

from .helpers import (
    get_orig_function,
    get_param_value_from_func_call,
    get_func_sourcecode,
    resolve_dependencies,
    clean_cachedir_old_entries,
)

_logger = logging.getLogger(__name__)

def catch(logger):
    """
    Decorator to catch an exception and don't raise it.
    Logs information if a decorator failed.

    Note:
        We don't want possible exceptions during logging to be raised.
        This is used to decorate any function that gets executed
        before or after the execution of the decorated function.
    """
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            try:
                return func(*args, **kwargs)
            except Exception:
                logger.warning(f"Exception raised in decorator: {func.__name__}")

        return wrapper
    return decorator

def _get_dfs(*args, **kwargs):
    values = args + tuple(kwargs.values())
    return [value for value in values if isinstance(value, pd.DataFrame)]

@catch(_logger)
def _get_dfs_shapes(*args, **kwargs):
    return [df.shape for df in _get_dfs(*args, **kwargs)]

@catch(_logger)
def _log_shapes(logger, func_name, input_shapes, output_shapes):
    logger.info(f"{func_name} - {input_shapes} -> {output_shapes}")

@catch(_logger)
def _log_time(logger, func_name, start, end):
    duration = (end - start) * 1000
    logger.info(f"{func_name} - time: {duration:0.1f} ms")

@catch(_logger)
def _log_message(logger, func_name, message):
    logger.info(f"{func_name} - {message}")

def log_message(logger, message=""):
    """
    Decorator to log a message before executing a function
    """
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            _log_message(logger, func.__name__, message)
            result = func(*args, **kwargs)
            return result
        return wrapper
    return decorator

def log_time(logger):
    """
    Decorator to log the execution time of a function
    """
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            start = time.time()
            result = func(*args, **kwargs)
            end = time.time()
            _log_time(logger, func.__name__, start, end)
            return result
        return wrapper
    return decorator

def log_shapes(logger):
    """
    Decorator to log the shapes of input and output dataframes

    It considers all the dataframes passed either as arguments or keyword arguments as inputs
    and all the dataframes returned as outputs.
    """
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            input_shapes = _get_dfs_shapes(*args, **kwargs)
            result = func(*args, **kwargs)
            output_shapes = _get_dfs_shapes(result)
            _log_shapes(logger, func.__name__, input_shapes, output_shapes)
            return result
        return wrapper
    return decorator

def log(logger=None, start_message='Starting...', end_message='Done...'):
    """
    Basic log decorator
    Can be used as :
    - @log (with default logger)
    - @log(mylogger)
    - @log(start_message='Hello !", logger=mylogger, end_message='Bye !')
    """
    def actual_log(f, real_logger=logger):
        logger = real_logger or _logger

        @wraps(f)
        def timed(*args, **kwargs):
            logger.info(f'{f.__name__} - {start_message}')
            start = time.time()
            res = f(*args, **kwargs)
            end = time.time()
            logger.info(f'{f.__name__} - {end_message} (took {end - start:.2f}s)')
            return res

        return timed

    if callable(logger):
        return actual_log(logger, real_logger=None)
    return actual_log

def domain(domain_name):
    """
    Allow to apply a function f(df: DataFrame) -> DataFrame) on dfs by specifying the key
    E.g instead of writing:
        def process_domain1(dfs):
            df = dfs['domain1']
            # actual process
            dfs['domain1'] = df
            return dfs

    You can write:
        @domain('domain1')
        def process_domain1(df):
            #actual process
            return df
    """
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            dfs, *args = args
            if not isinstance(dfs, dict):
                raise TypeError(f'{dfs} is not a dict')
            df = dfs.pop(domain_name)
            df = func(df, *args, **kwargs)
            return {domain_name: df, **dfs}

        return wrapper

    return decorator

# ~~~ @cache decorator related stuff ~~~

def cache(  # noqa: C901
    requires=None,
    disabled=False,
    applied_on_method=False,
    check_param=True,
    limit=None
):
    """ Avoid to recompute a function if its parameters and its source code doesnt have changed.

        Args:
            requires: list of dependencies (functions or function names)
            disabled (bool): disable the cache mecanism for this function (useful if you
                                 only want to use the dependency mecanism)
            applied_on_method (bool): ignore the first argument (useful to ignore "self")
            check_param (True, False or a str): the name of the parameter to check.
                                                    False to not check any of them.
                                                    True (default) to check all of them.
            limit (int or None): number of cache entries to keep (no limit by default)
    """
    if not requires:
        requires = []
    elif isinstance(requires, collections.Callable):
        requires = [requires]

    if not isinstance(check_param, (bool, str)):
        raise TypeError("'check_param' must be a str (name of the param to check) or a bool")
    if limit is not None and not isinstance(limit, int):
        raise TypeError("'limit' must be an int (number of cache entries to keep) or None")

    # We keep data in the function attributes so that this data
    # is not erased between two calls:
    if not hasattr(cache, 'funcs_references'):
        cache.funcs_references = {}  # dict of {function_name -> function_object (or None)}
    if not hasattr(cache, 'dependencies'):
        cache.dependencies = {}  # dict of {function_name -> [list of function names]}
    if not hasattr(cache, 'memories'):
        cache.memories = {}  # dict of {thread_id -> joblib.Memory object}

    def decorator(func):
        """ This code is executed when the augment module is read (when decorator is applied).
            Here we populate cache.funcs_references and cache.dependencies to use them later. """
        cache.funcs_references[func.__name__] = get_orig_function(func)
        dependencies_names = []
        for requirement in requires:
            if isinstance(requirement, collections.Callable):
                req_name = requirement.__name__
                cache.funcs_references[req_name] = get_orig_function(requirement)
            elif requirement not in cache.funcs_references:
                req_name = requirement
                cache.funcs_references[req_name] = None
            dependencies_names.append(req_name)

        cache.dependencies[func.__name__] = dependencies_names

        @wraps(func)
        def wrapper(*args, **kwargs):
            """ This code is executed when a decorated function is actually executed.
                It uses the previously built dependency tree (see above). """
            current_memory = cache.memories.get(current_thread().name)
            if disabled is True or current_memory is None:
                return func(*args, **kwargs)

            # if cache is enabled, we compute the md5 hash of the concatenated source codes
            # of all the dependencies.
            concatenated_source_code = ''
            dependencies = resolve_dependencies(func.__name__, cache.dependencies)
            for func_name in dependencies:
                function = cache.funcs_references[func_name]
                if function is None:
                    raise Exception(f"Can't get source code of function '{func_name}'")
                source_code = get_func_sourcecode(function)
                concatenated_source_code += source_code
            md5_hash = md5(str.encode(concatenated_source_code)).hexdigest()

            # Add extra parameters so that joblib checks they didnt have changed:
            tmp_extra_kwargs = {
                '__func_dependencies_hash__': md5_hash,
                '__original_func_name__': func.__name__,
            }

            if check_param is True:
                kwargs.update(tmp_extra_kwargs)

                if applied_on_method:
                    self_arg, args = args[0], args[1:]

                @wraps(func)
                def f(*args, **kwargs):
                    # delete the extra parameters that the underlying function doesnt expect:
                    for k in tmp_extra_kwargs.keys():
                        del kwargs[k]

                    if applied_on_method:
                        args = (self_arg,) + args
                    return func(*args, **kwargs)

                f = current_memory.cache(f)
                result = f(*args, **kwargs)
            else:
                if isinstance(check_param, str):
                    check_only_param_value = get_param_value_from_func_call(
                        param_name=check_param,
                        func=func,
                        call_args=args,
                        call_kwargs=kwargs,
                    )
                    tmp_extra_kwargs['__check_only__'] = check_only_param_value

                @wraps(func)
                def f(**tmp_extra_kwargs):
                    return func(*args, **kwargs)

                f = current_memory.cache(f)
                result = f(**tmp_extra_kwargs)

            if limit is not None:
                clean_cachedir_old_entries(f.store_backend, func.__name__, limit)

            return result

        return wrapper

    return decorator

method_cache = partial(cache, applied_on_method=True)

def setup_cachedir(cachedir, mmap_mode=None, bytes_limit=None):
    """ This function injects a joblib.Memory object in the cache() function
        (in a thread-specific slot of its 'memories' attribute). """
    if not hasattr(cache, 'memories'):
        cache.memories = {}

    memory = joblib.Memory(
        location=cachedir,
        verbose=0,
        mmap_mode=mmap_mode,
        bytes_limit=bytes_limit,
    )
    cache.memories[current_thread().name] = memory
    return memory


from typing import List

import pandas as pd

def melt(
        df,
        id: List[str],
        value: List[str],
        dropna=False
):
    """
    A melt will transform a dataset by creating a column "variable" and a column "value".
    This function is useful to transform a dataset into a format where one or more columns
    are identifier variables, while all other columns, considered measured
    variables (value_vars), are “unpivoted” to the row axis, leaving just two
    non-identifier columns, `"variable"` and `"value"`.

    ---

    ### Parameters

    *mandatory :*
    - `id` (*list of str*): names of the columns that must be kept in column.
    - `value` (*list of str*): names of the columns that will be transformed in long format (in rows).

    *optional :*
    - `dropna` (*boolean*): It allows you to drop missing values.

    ---

    ### Example

    **Input**

    | my_label | my_value | my_column_1 | my_column_2 | info_1 | info_2 | info_3 |
    |:--------:|:--------:|:-----------:|:-----------:|:------:|:------:|:------:|
    |   toto   |    10    |     S45     |    Lalaland |   10   |   20   |  None  |

    ```cson
    melt:
      id: ['my_label', 'my_value' 'my_column_1', 'my_colum_2']
        value: ['info_1', 'info_2', 'info_3']
        dropna: true
    ```

    **Ouput**

    | my_label | my_value | my_column_1 | my_column_2 | variable | value  |
    |:--------:|:--------:|:-----------:|:-----------:|:--------:|:------:|
    |   toto   |    10    |     S45     |    Lalaland |  info_1  |   10   |
    |   toto   |    10    |     S45     |    Lalaland |  info_2  |   20   |
    """
    df = df[(id + value)]
    df = pd.melt(df, id_vars=id, value_vars=value)
    if dropna:
        df = df.dropna(subset=['value'])

    return df


from typing import Dict

def rename(
        df,
        values: Dict[str, Dict[str, str]] = None,
        columns: Dict[str, Dict[str, str]] = None,
        locale: str = None
):
    """
    Replaces data values and column names according to the locale

    ---

    ### Parameters

    - `values` (optional: dict):
        - key: term to be replaced
        - value:
            - key: the locale e.g. 'en' or 'fr'
            - value: term's translation
    - `columns` (optional: dict):
        - key: columns name to be replaced
        - value:
            - key: the locale e.g. 'en' or 'fr'
            - value: column name's translation
    - `locale` (optional: str): the locale you want to use.
      By default the client locale is used.

    ---

    ### Example

    **Input**

    | label | value |
    |:----------------:|:-----:|
    | France | 100 |
    | Europe wo France | 500 |

    ```cson
    rename:
      values:
        'Europe wo France':
          'en': 'Europe excl. France'
          'fr': 'Europe excl. France'
      columns:
        'value':
          'en': 'revenue'
          'fr': 'revenue'
    ```

    **Output**

    | label | revenue |
    |:-------------------:|:-------:|
    | France | 100 |
    | Europe excl. France | 500 |

    """
    if values:
        to_replace = list(values.keys())
        value = [values[term][locale] for term in values]
        df = df.replace(to_replace=to_replace, value=value)
    if columns:
        _keys = list(columns.keys())
        _values = [column[locale] for column in columns.values()]
        columns = dict(list(zip(_keys, _values)))
        df = df.rename(columns=columns)
    return df


from typing import List

from toucan_data_sdk.utils.helpers import (
    check_params_columns_duplicate,
    ParamsValueError
)

def compute_cumsum(
    df,
    id_cols: List[str],
    reference_cols: List[str],
    value_cols: List[str],
    new_value_cols: List[str] = None,
    cols_to_keep: List[str] = None
):
    """
    Compute cumsum for a group of columns.

    ---

    ### Parameters

    *mandatory :*
    - `id_cols` (*list*): the columns id to create each group
    - `reference_cols` (*list*): the columns to order the cumsum
    - `value_cols` (*list*): the columns to cumsum

    *optional :*
    - `new_value_cols` (*list*): the new columns with the result cumsum
    - `cols_to_keep` (*list*): other columns to keep in the dataset.
      This option can be used if there is only one row by group [id_cols + reference_cols]

    ---

    ### Example

    **Input**

    MONTH | DAY | NAME | VALUE | X
    :---:|:---:|:--:|:---:|:---:
     1   |   1 |   A  |  1 | lo
     2   |   1 |   A  |  1 | lo
     2   |  15 |   A  |  1 | la
     1   |  15 |   B  |  1 | la

    ```cson
    compute_cumsum:
      id_cols: ['NAME']
      reference_cols: ['MONTH', 'DAY']
      cumsum_cols: ['VALUE']
      cols_to_keep: ['X']
    ```

    **Output**

    NAME | MONTH | DAY | X | VALUE
    :---:|:---:|:--:|:---:|:---:
     A  |   1  |    1 | lo  |    1
     A  |   2  |    1 | la  |    2
     A  |   2  |   15 | lo  |    3
     B  |   1  |   15 | la  |    1
    """
    if cols_to_keep is None:
        cols_to_keep = []

    if new_value_cols is None:
        new_value_cols = value_cols
    if len(value_cols) != len(new_value_cols):
        raise ParamsValueError('`value_cols` and `new_value_cols` needs '
                               'to have the same number of elements')

    check_params_columns_duplicate(id_cols + reference_cols + cols_to_keep + value_cols)

    levels = list(range(0, len(id_cols)))

    df = df.groupby(id_cols + reference_cols + cols_to_keep).sum()
    df[new_value_cols] = df.groupby(level=levels)[value_cols].cumsum()

    return df.reset_index()


import itertools
from typing import Dict, List, Union

import pandas as pd

def combine_columns_aggregation(
        df,
        id_cols: List[str],
        cols_for_combination: Dict[str, str],
        agg_func: Union[str, List[str], Dict[str, str]] = 'sum'
):
    """
    Aggregates data to reproduce "All" category for requester

    ---

    ### Parameters

    *mandatory :*
    - `id_cols` (*list*): the columns id to group
    - `cols_for_combination` (*dict*): colums corresponding to
       the filters as key and their default value as value

    *optional :*
    - `agg_func` (*str*, *list* or *dict*): the function(s) to use for aggregating the data.
       Accepted combinations are:
       - string function name
       - list of functions and/or function names, e.g. [np.sum, 'mean']
       - dict of axis labels -> functions, function names or list of such.
    """
    requesters_cols = list(cols_for_combination.keys())
    requester_combination = [
        list(item) for i in range(0, len(requesters_cols) + 1)
        for item in itertools.combinations(requesters_cols, i)]
    dfs_result = []
    for comb in requester_combination:
        df_tmp = df.groupby(id_cols + comb).agg(agg_func).reset_index()
        for key in (set(cols_for_combination.keys()) - set(comb)):
            df_tmp[key] = cols_for_combination[key]
        dfs_result.append(df_tmp)

    return pd.concat(dfs_result, sort=False, ignore_index=True)


import inspect
import linecache
import re
import shutil
from pathlib import Path
from typing import List

from joblib._store_backends import StoreBackendBase
from slugify import slugify as _slugify
import logging

logger = logging.getLogger(__name__)

def get_orig_function(f):
    """ Make use of the __wrapped__ attribute to find the original function
        of a decorated function. """
    try:
        while True:
            f = f.__wrapped__
    except AttributeError:
        return f

def get_param_value_from_func_call(param_name, func, call_args, call_kwargs):
    """
    Get the value of a function's parameter based on its signature
    and the call's args and kwargs.

    Example:
        >>> def foo(a, b, c=3, d=4):
        ...    pass
        ...
        >>> # what would be the value of "c" when calling foo(1, b=2, c=33) ?
        >>> get_param_value_from_func_call('c', foo, [1], {'b': 2, 'c': 33})
        33
    """
    signature = inspect.signature(func)
    params_list = signature.parameters.keys()
    if param_name not in params_list:
        raise TypeError(f"'{param_name}' not found in {func.__name__}"
                        f"parameters list ([{params_list}])")
    call = signature.bind(*call_args, **call_kwargs)
    call.apply_defaults()
    return call.arguments[param_name]

def get_func_sourcecode(func):
    """
    Try to get sourcecode using standard inspect.getsource().
    If the function comes from a module which has been created dynamically
    (not from the filesystem), then it tries to read the sourcecode on the
    filesystem anyway.
    WARNING: can do weird things if the filesystem code slightly differs from
             the original module code.
    """

    def getsource(func):
        lines, lnum = getsourcelines(func)
        return ''.join(lines)

    def getsourcelines(func):
        lines, lnum = findsource(func)
        return inspect.getblock(lines[lnum:]), lnum + 1

    def findsource(func):
        file = getfile(func)  # file path
        module = inspect.getmodule(func, file)
        lines = linecache.getlines(file, module.__dict__)
        code = func.__code__
        lnum = code.co_firstlineno - 1
        pat = re.compile(r'^(\s*def\s)|(\s*async\s+def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)')
        while lnum > 0:
            if pat.match(lines[lnum]):
                break
            lnum = lnum - 1  # pragma: no cover
        return lines, lnum

    def getfile(func):
        module = inspect.getmodule(func)
        return module.__file__

    try:
        return inspect.getsource(func)
    except Exception:
        return getsource(func)

def check_params_columns_duplicate(cols_name: List[str]) -> bool:
    params = [column for column in cols_name if column is not None]
    if len(set(params)) != len(params):
        duplicates = set([x for x in params if params.count(x) > 1])
        raise ParamsValueError(
            f'Duplicate declaration of column(s) {duplicates} in the parameters')
    else:
        return True

def slugify(name, separator='-'):
    """Returns a slugified name (we allow _ to be used)"""
    return _slugify(name, regex_pattern=re.compile('[^-_a-z0-9]+'), separator=separator)

def resolve_dependencies(func_name, dependencies):
    """ Given a function name and a mapping of function dependencies,
        returns a list of *all* the dependencies for this function. """

    def _resolve_deps(func_name, func_deps):
        """ Append dependencies recursively to func_deps (accumulator) """
        if func_name in func_deps:
            return

        func_deps.append(func_name)
        for dep in dependencies.get(func_name, []):
            _resolve_deps(dep, func_deps)

    func_deps = []
    _resolve_deps(func_name, func_deps)
    return sorted(func_deps)

def clean_cachedir_old_entries(cachedir: StoreBackendBase, func_name: str, limit: int) -> int:
    """Remove old entries from the cache"""
    if limit < 1:
        raise ValueError("'limit' must be greater or equal to 1")

    cache_entries = get_cachedir_entries(cachedir, func_name)
    cache_entries = sorted(cache_entries, key=lambda e: e.last_access, reverse=True)
    cache_entries_to_remove = cache_entries[limit:]
    for entry in cache_entries_to_remove:
        shutil.rmtree(entry.path, ignore_errors=True)

    return len(cache_entries_to_remove)

def get_cachedir_entries(cachedir: StoreBackendBase, func_name: str) -> list:
    entries = cachedir.get_items()
    return [e for e in entries if Path(e.path).parent.name == func_name]

class ParamsValueError(Exception):
    """
    Exception raised when some parameters value are wrong
    """


import pandas as pd
from typing import List

def roll_up(
        df,
        levels: List[str],
        groupby_vars: List[str],
        extra_groupby_cols: List[str] = None,
        var_name: str = 'type',
        value_name: str = 'value',
        agg_func: str = 'sum',
        drop_levels: List[str] = None
):
    """
    Creates aggregates following a given hierarchy

    ---

    ### Parameters

    *mandatory :*
    - `levels` (*list of str*): name of the columns composing the hierarchy (from the top to the bottom level).
    - `groupby_vars` (*list of str*): name of the columns with value to aggregate.
    - `extra_groupby_cols` (*list of str*) optional: other columns used to group in each level.

    *optional :*
    - `var_name` (*str*) : name of the result variable column. By default, `“type”`.
    - `value_name` (*str*): name of the result value column. By default, `“value”`.
    - `agg_func` (*str*): name of the aggregation operation. By default, `“sum”`.
    - `drop_levels` (*list of str*): the names of the levels that you may want to discard from the output.

    ---

    ### Example

    **Input**

    |    Region |     City |  Population |
    |:---------:|:--------:|:-----------:|
    |       Idf |     Panam|         200 |
    |       Idf |   Antony |          50 |
    |      Nord |    Lille |          20 |

    ```cson
    roll_up:
      levels: ["Region", "City"]
      groupby_vars: "Population"
    ```

    **Output**

    |    Region |     City |  Population |    value |   type |
    |:---------:|:--------:|:-----------:|:--------:|:------:|
    |       Idf |     Panam|         200 |    Panam |   City |
    |       Idf |   Antony |          50 |   Antony |   City |
    |      Nord |    Lille |          20 |    Lille |   City |
    |       Idf |      Nan |         250 |      Idf | Region |
    |      Nord |      Nan |          20 |     Nord | Region |
    """
    dfs = list()
    groupby_cols_cpy = list(levels)
    levels_cpy = list(levels)
    levels_cpy.reverse()

    extra_groupby_cols = extra_groupby_cols or []
    drop_levels = drop_levels or []
    previous_level = None
    for top_level in levels_cpy:
        # Aggregation
        gb_df = getattr(
            df.groupby(groupby_cols_cpy + extra_groupby_cols)[groupby_vars],
            agg_func)().reset_index()

        # Melt-like columns
        gb_df[var_name] = top_level
        gb_df[value_name] = gb_df[top_level]
        dfs.append(gb_df)
        if previous_level in drop_levels:
            del dfs[-2]
        previous_level = top_level

        # Remove one level each time in the groupby: lowest level column needs
        # a groupby with every levels, the next level needs every one except
        # the lowest, etc. until the top level column that needs only itself
        # inside the groupby.
        groupby_cols_cpy.pop()
    return pd.concat(dfs, sort=False).reset_index()


from typing import List, Union

def argmax(df, column: str, groups: Union[str, List[str]] = None):
    """
    Keep the row of the data corresponding to the maximal value in a column

    ---

    ### Parameters

    *mandatory :*
    - `column` (*str*): name of the column containing the value you want to keep the maximum

    *optional :*
    - `groups` (*str or list(str)*): name of the column(s) used for 'groupby' logic
    (the function will return the argmax by group)

    ---

    ### Example

    **Input**

    | variable |   wave  |  year    | value |
    |:--------:|:-------:|:--------:|:-----:|
    |   toto   |  wave 1 |  2014    |  300  |
    |   toto   |  wave 1 |  2015    |  250  |
    |   toto   |  wave 1 |  2016    |  450  |

    ```cson
    argmax:
      column: 'year'
    ```

    **Output**

    | variable |   wave  |  year    | value |
    |:--------:|:-------:|:--------:|:-----:|
    |   toto   |  wave 1 |  2016    |  450  |
    """
    if groups is None:
        df = df[df[column] == df[column].max()].reset_index(drop=True)
    else:
        group_max = df.groupby(groups)[column].transform('max')
        df = (df
              .loc[df[column] == group_max, :]
              .drop_duplicates()
              .reset_index(drop=True)
              )
    return df

def argmin(df, column: str, groups: Union[str, List[str]] = None):
    """
    Keep the row of the data corresponding to the minimal value in a column

    ---

    ### Parameters

    *mandatory :*
    - `column` (str): name of the column containing the value you want to keep the minimum

    *optional :*
    - `groups` (*str or list(str)*): name of the column(s) used for 'groupby' logic
    (the function will return the argmax by group)
    ---

    ### Example

    **Input**

    | variable |   wave  |  year    | value |
    |:--------:|:-------:|:--------:|:-----:|
    |   toto   |  wave 1 |  2014    |  300  |
    |   toto   |  wave 1 |  2015    |  250  |
    |   toto   |  wave 1 |  2016    |  450  |

    ```cson
    argmin:
      column: 'year'
    ]
    ```

    **Output**

    | variable |   wave  |  year    | value |
    |:--------:|:-------:|:--------:|:-----:|
    |   toto   |  wave 1 |  2015    |  250  |
    """
    if groups is None:
        df = df[df[column] == df[column].min()].reset_index(drop=True)
    else:
        group_min = df.groupby(groups)[column].transform('min')
        df = (df
              .loc[df[column] == group_min, :]
              .drop_duplicates()
              .reset_index(drop=True)
              )
    return df


from numpy import nan

def fillna(df, column: str, value=None,  column_value=None):
    """
    Can fill NaN values from a column with a given value or a column

    ---

    ### Parameters

    - `column` (*str*): name of column you want to fill
    - `value`: NaN will be replaced by this value
    - `column_value`: NaN will be replaced by value from this column

    *NOTE*: You must set either the 'value' parameter or the 'column_value' parameter

    ---

    ### Example

    **Input**

    | variable |   wave  |  year    | my_value |
    |:--------:|:-------:|:--------:|:--------:|
    |   toto   |  wave 1 |  2014    |  300     |
    |   toto   |  wave 1 |  2015    |          |
    |   toto   |  wave 1 |  2016    |  450     |

    ```cson
    fillna:
      column: 'my_value'
      value: 0
    ```

    **Output**

    | variable |   wave  |  year    | my_value |
    |:--------:|:-------:|:--------:|:--------:|
    |   toto   |  wave 1 |  2014    |  300     |
    |   toto   |  wave 1 |  2015    |    0     |
    |   toto   |  wave 1 |  2016    |  450     |
    """
    if column not in df.columns:
        df[column] = nan

    if value is not None and column_value is not None:
        raise ValueError('You cannot set both the parameters value and column_value')

    if value is not None:
        df[column] = df[column].fillna(value)

    if column_value is not None:
        if column_value not in df.columns:
            raise ValueError(f'"{column_value}" is not a valid column name')
        df[column] = df[column].fillna(df[column_value])

    return df


import pandas as pd
from typing import Dict

def date_requester_generator(
        df: pd.DataFrame,
        date_column: str,
        frequency: str,
        date_column_format: str = None,
        format: str = '%Y-%m-%d',
        granularities: Dict[str, str] = None,
        others_format: Dict[str, str] = None,
        times_delta: Dict[str, str] = None
) -> pd.DataFrame:
    """
    From a dataset containing dates in a column, return a dataset
    with at least 3 columns :
    - "DATE" : Label of date
    - "DATETIME" : Date in datetime dtype
    - "GRANULARITY" : Granularity of date

    ---

    ### Parameters

    *mandatory :*
    - `date_column` (*str*): name of column containing the date in the dataframe
    - `frequency` (*str*): see [pandas doc](
    http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases)

    *optional :*
    - `date_column_format` (*str*): format of the date in date_column
    - `format` (*str*): format of the date (see [pandas doc](
        https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior))
        By default, the format is set to `'%d/%m/%Y'`
        **WARNING**: only use if `granularities` is None.
    - `granularities` (*dict*):
        - key (*str*): name of the granularity
        - value (*str*): Format of the granularity e.g. '%d/%m/%Y' (see [pandas doc](
            https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior))
    - `others_format` (*dict*) : Add new columns for each key
        - key (*str*) : name of the column
        - value (*str*): format of the granularity e.g. '%d/%m/%Y' (see [pandas doc](
            https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior))
    - `times_delta` (*dict*) : Add new columns for each key
        - key (*str*) : name of the column
        - value (*str*): time delta (e.g. '+1 day', '+3 day', '-4 month')

    ---

    ### Example

    **Input**

    date | kpi
    :---:|:-----:
    2018-01-01 | 1
    2018-01-05 | 2
    2018-01-04 | 3
    2018-01-03 | 4
    2018-01-02 | 5

    ```cson
    date_requester_generator:
      date_column: 'date'
      frequency: 'D'
      granularities:
        'day': '%d/%m/%Y'
        'Semaine': '%W'
      others_format:
        'year': '%Y'
    ```

    **Ouput**

    DATE  |  DATETIME  | GRANULARITY | year
    :---------:|:----------:|:-----------:|:---:
    01/01/2018 | 2018-01-01 |         day | 2018
    02/01/2018 | 2018-01-02 |         day | 2018
    03/01/2018 | 2018-01-03 |         day | 2018
    04/01/2018 | 2018-01-04 |         day | 2018
    05/01/2018 | 2018-01-05 |         day | 2018
    01 | 2018-01-01 |     Semaine | 2018
    """

    start_date = pd.to_datetime(df[date_column], format=date_column_format).min()
    end_date = pd.to_datetime(df[date_column], format=date_column_format).max()

    granularities = granularities or {'date': format}
    others_format = others_format or {}
    times_delta = times_delta or {}

    # Base DataFrame
    columns_list = ['DATE', 'DATETIME', 'GRANULARITY', *others_format, *times_delta]
    result_df = {col_name: [] for col_name in columns_list}

    # Generate the range
    date_range = pd.date_range(start=start_date, end=end_date, freq=frequency)

    for granularity_name, granularity_format in granularities.items():
        date_range_label = date_range.strftime(granularity_format)
        a = list(set(date_range_label))
        index_unique = list(set([a.index(x) for x in date_range_label]))
        date_range_datetime = date_range[index_unique]
        date_range_label = date_range_label.unique()

        result_df['DATE'] += list(date_range_label)
        result_df['DATETIME'] += list(date_range_datetime)
        result_df['GRANULARITY'] += [granularity_name] * len(date_range_label)

        for col_name, other_format in others_format.items():
            result_df[col_name] += list(date_range_datetime.strftime(other_format))

        for col_name, time_delta in times_delta.items():
            result_df[col_name] += list((date_range_datetime + pd.Timedelta(time_delta))
                                        .strftime(granularity_format))

    return pd.DataFrame(result_df)


"""date filtering helpers."""

from calendar import monthrange
from datetime import date, datetime, timedelta
import re
from uuid import uuid4

import pandas as pd

TIMEDELTA_RGX = re.compile(r'\s*(?P<num>\d+)\s*(?P<unit>\w+)$')

def _norm_date(datestr: str, date_fmt: str) -> date:
    """normalize symbolic date values (e.g. 'TODAY')

    Convert a symbolic value in a valid date.
    Currenlty known symbolic values are 'TODAY', 'YESTERDAY' and 'TOMORROW'.

    NOTE: This function will return `date` (not `datetime`) instances.

    Parameters:
        `datestr`: the date to parse, formatted as `date_fmt`
        `date_fmt`: expected output date format

    Returns:
        The interpreted date as a datetime.datetime object.
        If `datestr` doesn't match any of the known symbolic names, it just parses it.

    """
    try:
        days = {'TODAY': 0, 'YESTERDAY': -1, 'TOMORROW': 1}[datestr.upper()]
        return date.today() + pd.Timedelta(days=days)
    except KeyError:
        return datetime.strptime(datestr, date_fmt).date()

def add_offset(dateobj, hr_offset: str, sign: str):
    """add a human readable offset to `dateobj` and return corresponding date.

    rely on `pandas.Timedelta` and add the following extra shortcuts:
    - "w", "week" and "weeks" for a week (i.e. 7days)
    - "month', "months" for a month (i.e. no day computation, just increment the month)
    - "y", "year', "years" for a year (i.e. no day computation, just increment the year)
    """
    sign_coeff = 1 if sign == '+' else -1
    try:
        return dateobj + sign_coeff * pd.Timedelta(hr_offset)
    except ValueError:
        # pd.Timedelta could not parse the offset, let's try harder
        match = TIMEDELTA_RGX.match(hr_offset)
        if match is not None:
            groups = match.groupdict()
            unit = groups['unit'].lower()[0]
            num = sign_coeff * int(groups['num'])
            # is it a week ?
            if unit == 'w':
                return dateobj + num * timedelta(weeks=1)
            # or a month ?
            if unit == 'm':
                return add_months(dateobj, num)
            # or a year ?
            if unit == 'y':
                return add_years(dateobj, num)
        # we did what we could, just re-raise the original exception
        raise

def add_months(dateobj, nb_months: int):
    """return `dateobj` + `nb_months`

    If landing date doesn't exist (e.g. february, 30th), return the last
    day of the landing month.

    >>> add_months(date(2018, 1, 1), 1)
    datetime.date(2018, 1, 1)
    >>> add_months(date(2018, 1, 1), -1)
    datetime.date(2017, 12, 1)
    >>> add_months(date(2018, 1, 1), 25)
    datetime.date(2020, 2, 1)
    >>> add_months(date(2018, 1, 1), -25)
    datetime.date(2015, 12, 1)
    >>> add_months(date(2018, 1, 31), 1)
    datetime.date(2018, 2, 28)
    """
    nb_years, nb_months = divmod(nb_months, 12)
    month = dateobj.month + nb_months
    if month > 12:
        nb_years += 1
        month -= 12
    year = dateobj.year + nb_years
    lastday = monthrange(year, month)[1]
    return dateobj.replace(year=year, month=month, day=min(lastday, dateobj.day))

def add_years(dateobj, nb_years):
    """return `dateobj` + `nb_years`

    If landing date doesn't exist (e.g. february, 30th), return the last
    day of the landing month.

    >>> add_years(date(2018, 1, 1), 1)
    datetime.date(2019, 1, 1)
    >>> add_years(date(2018, 1, 1), -1)
    datetime.date(2017, 1, 1)
    >>> add_years(date(2020, 2, 29), 1)
    datetime.date(2021, 2, 28)
    >>> add_years(date(2020, 2, 29), -1)
    datetime.date(2019, 2, 28)
    """
    year = dateobj.year + nb_years
    lastday = monthrange(year, dateobj.month)[1]
    return dateobj.replace(year=year, day=min(lastday, dateobj.day))

def parse_date(datestr: str, date_fmt: str) -> date:
    """parse `datestr` and return corresponding date object.

    `datestr` should be a string matching `date_fmt` and parseable by `strptime`
    but some offset can also be added using `(datestr) + OFFSET` or `(datestr) -
    OFFSET` syntax. When using this syntax, `OFFSET` should be understable by
    `pandas.Timedelta` (cf.
    http://pandas.pydata.org/pandas-docs/stable/timedeltas.html) and `w`, `week`
    `month` and `year` offset keywords are also accepted. `datestr` MUST be wrapped
    with parenthesis.

    Additionally, the following symbolic names are supported: `TODAY`,
    `YESTERDAY`, `TOMORROW`.

    Example usage:

    >>> parse_date('2018-01-01', '%Y-%m-%d') datetime.date(2018, 1, 1)
    parse_date('(2018-01-01) + 1day', '%Y-%m-%d') datetime.date(2018, 1, 2)
    parse_date('(2018-01-01) + 2weeks', '%Y-%m-%d') datetime.date(2018, 1, 15)

    Parameters: `datestr`: the date to parse, formatted as `date_fmt`
        `date_fmt`: expected date format

    Returns: The `date` object. If date could not be parsed, a ValueError will
        be raised.
    """
    rgx = re.compile(r'\((?P<date>.*)\)(\s*(?P<sign>[+-])(?P<offset>.*))?$')
    datestr = datestr.strip()
    match = rgx.match(datestr)
    # if regexp doesn't match, date must match the expected format
    if match is None:
        return _norm_date(datestr, date_fmt)
    datestr = match.group('date').strip()
    dateobj = _norm_date(datestr, date_fmt)
    offset = match.group('offset')
    if offset:
        return add_offset(dateobj, offset, match.group('sign'))
    return dateobj

def filter_by_date(
    df,
    date_col: str,
    date_format: str = '%Y-%m-%d',
    start: str = None,
    stop: str = None,
    atdate: str = None
):
    """
    Filter dataframe your data by date.

    This function will interpret `start`, `stop` and `atdate` and build
    the corresponding date range. The caller must specify either:

    - `atdate`: keep all rows matching this date exactly,
    - `start`: keep all rows matching this date onwards.
    - `stop`: keep all rows matching dates before this one.
    - `start` and `stop`: keep all rows between `start` and `stop`,

    Any other combination will raise an error. The lower bound of the date range
    will be included, the upper bound will be excluded.

    When specified, `start`, `stop` and `atdate` values are expected to match the
    `date_format` format or a known symbolic value (i.e. 'TODAY', 'YESTERDAY' or 'TOMORROW').

    Additionally, the offset syntax "(date) + offset" is also supported (Mind
    the parenthesis around the date string). In that case, the offset must be
    one of the syntax supported by `pandas.Timedelta` (see [pandas doc](
    http://pandas.pydata.org/pandas-docs/stable/timedeltas.html))

    ---

    ### Parameters

    *mandatory :*
    - `date_col` (*str*): the name of the dataframe's column to filter on

    *optional :*
    - `date_format` (*str*): expected date format in column `date_col` (see [available formats](
    https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior)
    - `start` (*str*): if specified, lower bound (included) of the date range
    - `stop` (*str*): if specified, upper bound (excluded) of the date range
    - `atdate` (*str*): if specified, the exact date we're filtering on
    """
    mask = None
    if start is None and stop is None and atdate is None:
        raise TypeError('either "start", "stop" or "atdate" must be specified')
    if start is not None and atdate is not None:
        raise TypeError('"start" and "atdate" are mutually exclusive')
    if stop is not None and atdate is not None:
        raise TypeError('"stop" and "atdate" are mutually exclusive')
    # add a new column that will hold actual date objects instead of strings.
    # This column is just temporary and will be removed before returning the
    # filtered dataframe.
    filtercol = str(uuid4())
    df[filtercol] = pd.to_datetime(df[date_col], format=date_format)
    if atdate is not None:
        mask = df[filtercol] == parse_date(atdate, date_format)
    elif start is not None and stop is not None:
        mask = ((df[filtercol] >= parse_date(start, date_format)) &
                (df[filtercol] < parse_date(stop, date_format)))
    elif stop is None:
        mask = df[filtercol] >= parse_date(start, date_format)
    elif start is None:
        mask = df[filtercol] < parse_date(stop, date_format)
    return df[mask].drop(filtercol, axis=1)


def replace(df, column: str, new_column: str = None, **kwargs):
    """
    Change the label of a value or a columns within your data source.
    (Similar to `rename` but does not have the notion of locale)

    ---

    ### Parameters

    *mandatory :*
    - `column` (*str*): name of the column to modify.
    - `to_replace` (*dict*): keys of this dict are old values pointing on substitute.

    *optional :*
    - `new_column` (*str*): name of the output column. By default the `column` arguments is modified.

    **Note**: extra parameters can be used (see [pandas doc](
    https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.replace.html))

    ---

    ### Example

    **Input**

    article | rating
    :------:|:------:
    book    | 1
    puzzle  | 3
    food    | 5

    We want to split the ratings in three categories: "good", "average" and "poor".

    ```cson
    replace:
      column: "rating"
      new_column: "rating_category"  # create a new column with replaced data
      to_replace:
        1: "poor"
        2: "poor"
        3: "average"
        4: "good"
        5: "good"
    ```

    **Ouput**

    article | rating | rating_category
    :------:|:------:|:--------------:
    book    | 1      | poor
    puzzle  | 3      | average
    food    | 5      | good
    """
    new_column = new_column or column
    df.loc[:, new_column] = df[column].replace(**kwargs)
    return df


from typing import Union, List

def percentage(
        df,
        column: str,
        group_cols: Union[str, List[str]] = None,
        new_column: str = None
):
    """
    Add a column to the dataframe according to the groupby logic on group_cols

    ---

    ### Parameters

    *mandatory :*
    - `column` (*str*): name of the desired column you need percentage on

    *optional :*
    - `group_cols` (*list*): names of columns for the groupby logic
    - `new_column` (*str*): name of the output column. By default `column` will be overwritten.

    ---

    **Input**

    | gender |    sport   | number |
    |:------:|:----------:|:------:|
    |  male  |   bicycle  |   17   |
    | female | basketball |   17   |
    |  male  | basketball |    3   |
    | female |  football  |    7   |
    | female |   running  |   30   |
    |  male  |   running  |   20   |
    |  male  |  football  |   21   |
    | female |   bicycle  |   17   |

    ```cson
    percentage:
      new_column: 'number_percentage'
      column: 'number'
      group_cols: ['sport']
    ```

    **Output**

    | gender |    sport   | number | number_percentage |
    |:------:|:----------:|:------:|:-----------------:|
    |  male  |   bicycle  |   17   |        50.0       |
    | female | basketball |   17   |        85.0       |
    |  male  | basketball |    3   |        15.0       |
    | female |  football  |    7   |        25.0       |
    | female |   running  |   30   |        60.0       |
    |  male  |   running  |   20   |        40.0       |
    |  male  |  football  |   21   |        75.0       |
    | female |   bicycle  |   17   |        50.0       |
    """
    new_column = new_column or column
    if group_cols is None:
        df[new_column] = 100. * df[column] / sum(df[column])
    else:
        df[new_column] = 100. * df[column] / df.groupby(group_cols)[column].transform(sum)
    return df


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from collections import OrderedDict
import inspect
import numpy as np
import theano
from theano import tensor as T
from deepy.trainers.optimize import logging
from deepy.core.env import FLOATX

def ada_family_core(params, gparams, learning_rate = 0.01, eps= 1e-6, rho=0.95, method="ADADELTA",
                        beta=0.0, gsum_regularization = 0.0001):
    """
    Optimize by SGD, AdaGrad, or AdaDelta.
    """

    _, _, _, args = inspect.getargvalues(inspect.currentframe())
    logging.info("ada_family_core: %s" % str(args.items()))
    free_parameters = []

    if method == "FINETUNING_ADAGRAD":
        method = "ADAGRAD"
        gsum_regularization = 0

    oneMinusBeta = 1 - beta

    gsums   = [theano.shared(np.zeros_like(param.get_value(borrow=True), dtype=FLOATX), name="gsum_%s" % param.name) if (method == 'ADADELTA' or method == 'ADAGRAD') else None for param in params]
    xsums   = [theano.shared(np.zeros_like(param.get_value(borrow=True), dtype=FLOATX), name="xsum_%s" % param.name) if method == 'ADADELTA' else None for param in params]

    # Fix for AdaGrad, init gsum to 1
    if method == 'ADAGRAD':
        for gsum in gsums:
            gsum.set_value(gsum.get_value() ** 0)

    updates = OrderedDict()
    # Updates
    for gparam, param, gsum, xsum in zip(gparams, params, gsums, xsums):

        if method == 'ADADELTA':
            updates[gsum] = rho * gsum + (1. - rho) * (gparam **2)
            dparam = -T.sqrt((xsum + eps) / (updates[gsum] + eps)) * gparam
            updates[xsum] =rho * xsum + (1. - rho) * (dparam **2)
            updates[param] = param * oneMinusBeta + dparam
        elif method == 'ADAGRAD':
            updates[gsum] = gsum + (gparam **2) - gsum_regularization * gsum
            updates[param] =  param * oneMinusBeta - learning_rate * (gparam / (T.sqrt(updates[gsum] + eps)))

        else:
            updates[param] = param * oneMinusBeta - gparam * learning_rate
    # Add free parameters
    if method == 'ADADELTA':
        free_parameters.extend(gsums + xsums)
    elif method == 'ADAGRAD':
        free_parameters.extend(gsums)
    # Check dtype
    for k in updates:
        if updates[k].dtype != FLOATX:
            updates[k] = updates[k].astype(FLOATX)
    return updates.items(), free_parameters


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import logging as loggers

import numpy as np
import theano
import theano.tensor as T
from theano.ifelse import ifelse

from deepy.utils import dim_to_var
from deepy.core.env import FLOATX, EPSILON
from deepy.trainers.util import wrap_core, multiple_l2_norm
from deepy.conf import TrainerConfig

logging = loggers.getLogger(__name__)

def optimize_updates(params, gradients, config=None, shapes=None):
    """
    General optimization function for Theano.
    Parameters:
        params - parameters
        gradients - gradients
        config - training config
    Returns:
        Theano updates
    :type config: deepy.TrainerConfig or dict
    """
    if config and isinstance(config, dict):
        config = TrainerConfig(config)

    # Clipping
    if config:
        clip_value = config.get("gradient_clipping", None)

        if clip_value:
            clip_constant = T.constant(clip_value, dtype=FLOATX)

            if config.avoid_compute_embed_norm:
                grad_norm = multiple_l2_norm([t[1] for t in zip(params, gradients) if not t[0].name.startswith("W_embed")])
            else:
                grad_norm = multiple_l2_norm(gradients)
            isnan = T.or_(T.isnan(grad_norm), T.isinf(grad_norm))
            multiplier = ifelse(grad_norm < clip_constant,
                                T.constant(1., dtype=FLOATX), clip_constant / (grad_norm + EPSILON))

            # Clip
            clipped_gradients = []
            for param, g in zip(params, gradients):
                g = multiplier * g
                if config.avoid_nan:
                    g = T.switch(isnan, np.float32(0.1) * param, g)
                if config.gradient_tolerance:
                    g = ifelse(grad_norm > config.gradient_tolerance, T.zeros_like(g) + EPSILON, g)
                clipped_gradients.append(g)

            gradients = clipped_gradients
    # Regularization
    if config and config.weight_l2:
        regularized_gradients = []
        for param, grad in zip(params, gradients):
            grad = grad + (2 * config.weight_l2 * param)
            regularized_gradients.append(grad)
        gradients = regularized_gradients

    # Avoid nan but not computing the norm
    # This is not recommended
    if config and config.avoid_nan and not config.gradient_clipping:
        logging.info("avoid NaN gradients")
        new_gradients = []
        for grad in gradients:
            new_grad = ifelse(T.isnan(grad).any(), T.zeros_like(grad) + EPSILON, grad)
            new_gradients.append(new_grad)
        gradients = new_gradients

    # Find method
    method = "SGD"
    if config:
        method = config.get("method", method).upper()
    # Get Function
    func = None
    if method in ["SGD", "ADAGRAD", "ADADELTA", "FINETUNING_ADAGRAD"]:
        from cores.ada_family import ada_family_core
        func = ada_family_core
    elif method == "ADAM":
        from cores.adam import adam_core
        func = adam_core
    elif method == "RMSPROP":
        from cores.rmsprop import rmsprop_core
        func = rmsprop_core
    elif method == "MOMENTUM":
        from cores.momentum import momentum_core
        func = momentum_core

    if not func:
        raise NotImplementedError("method '%s' is not supported" % method)

    logging.info("optimize method=%s parameters=%s" % (method, str(params)))

    free_parameters = []
    return_vals = wrap_core(func, config, params, gradients)
    if type(return_vals) == list and type(return_vals[0]) == list:
        updates, free_parameters = return_vals
    else:
        updates = return_vals

    # No free param recording
    if config and not config.record_free_params:
        free_parameters = []

    # Weight bound
    if config.weight_bound:
        logging.info("apply weight bound of %.2f" % config.weight_bound)
        new_updates = []
        for param, update_value in updates:
            bounded_value = (update_value * (T.abs_(update_value) <= config.weight_bound) +
                             config.weight_bound * (update_value > config.weight_bound) +
                             -config.weight_bound * (update_value < -config.weight_bound))
            new_updates.append((param, bounded_value))
        updates = new_updates
    return updates, free_parameters

def optimize_function(params, config=None):
    """
    Create a optimizing function receives gradients.
    Parameters:
        params - parameters
        config - training configuration
    Returns:
        updating function receives gradients
    """
    gs = [dim_to_var(p.ndim) for p in params]
    updates, _ = optimize_updates(params, gs, config)
    return theano.function(gs, [], updates=updates)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import time
import theano
import theano.tensor as T
from deepy.conf import TrainerConfig
from deepy.trainers.base import NeuralTrainer
from deepy.trainers.optimize import optimize_updates
from logging import getLogger
logging = getLogger(__name__)

class GeneralNeuralTrainer(NeuralTrainer):
    """
    General neural network trainer.
    """
    def __init__(self, network, method=None, config=None, annealer=None, validator=None):

        if method:
            logging.info("changing optimization method to '%s'" % method)
            if not config:
                config = TrainerConfig()
            elif isinstance(config, dict):
                config = TrainerConfig(config)
            config.method = method

        super(GeneralNeuralTrainer, self).__init__(network, config, annealer=annealer, validator=validator)

        self._learning_func = None

    def learn(self, *variables):
        if not self._learning_func:
            start_time = time.time()
            logging.info('compiling %s learning function', self.__class__.__name__)
            self._learning_func = self.learning_function()
            self._compile_time = time.time() - start_time
            logging.info("took {} seconds to compile".format(int(self._compile_time)))
        return self._learning_func(*variables)

    def _learning_updates(self):
        """
        Return updates in the training.
        """
        params = self.training_params()
        gradients = self.get_gradients(params)
        return self.optimization_updates(params, gradients)

    def training_params(self):
        """
        Get parameters to be optimized.
        """
        params = self.network.parameters
        # Freeze parameters
        if self.config.fixed_parameters:
            logging.info("fixed parameters: %s" % ", ".join(map(str, self.config.fixed_parameters)))
            params = [p for p in params if p not in self.config.fixed_parameters]
        return params

    def get_gradients(self, params):
        """
        Get gradients from given parameters.
        """
        return T.grad(self.cost, params)

    def optimization_updates(self, params, gradients):
        """
        Return updates from optimization.
        """
        updates, free_parameters = optimize_updates(params, gradients, self.config)
        self.network.free_parameters.extend(free_parameters)
        logging.info("Added %d free parameters for optimization" % len(free_parameters))
        return updates

    def learning_function(self):
        """
        Get the learning function.
        :param func:
        :return:
        """
        network_updates = list(self.network.updates) + list(self.network.training_updates)
        learning_updates = list(self._learning_updates())
        update_list = network_updates + learning_updates

        logging.info("network updates: %s" % " ".join(map(str, [x[0] for x in network_updates])))
        logging.info("learning updates: %s" % " ".join(map(str, [x[0] for x in learning_updates])))

        variables = self.network.input_variables + self.network.target_variables
        givens = None
        return theano.function(
            variables,
            map(lambda v: theano.Out(v, borrow=True), self.training_variables),
            updates=update_list, allow_input_downcast=True,
            mode=self.config.get("theano_mode", None),
            givens=givens)

class SGDTrainer(GeneralNeuralTrainer):
    """
    SGD trainer.
    """
    def __init__(self, network, config=None):
        super(SGDTrainer, self).__init__(network, config, "SGD")

class AdaDeltaTrainer(GeneralNeuralTrainer):
    """
    AdaDelta trainer.
    """
    def __init__(self, network, config=None):
        super(AdaDeltaTrainer, self).__init__(network, config, "ADADELTA")

class AdaGradTrainer(GeneralNeuralTrainer):
    """
    AdaGrad trainer.
    """
    def __init__(self, network, config=None):
        super(AdaGradTrainer, self).__init__(network, config, "ADAGRAD")

class FineTuningAdaGradTrainer(GeneralNeuralTrainer):
    """
    AdaGrad trainer.
    """
    def __init__(self, network, config=None):
        super(FineTuningAdaGradTrainer, self).__init__(network, config, "FINETUNING_ADAGRAD")

class AdamTrainer(GeneralNeuralTrainer):
    """
    AdaGrad trainer.
    """
    def __init__(self, network, config=None):
        super(AdamTrainer, self).__init__(network, config, "ADAM")

class RmspropTrainer(GeneralNeuralTrainer):
    """
    RmsProp trainer.
    """
    def __init__(self, network, config=None):
        super(RmspropTrainer, self).__init__(network, config, "RMSPROP")

class MomentumTrainer(GeneralNeuralTrainer):
    """
    Momentum trainer.
    """
    def __init__(self, network, config=None):
        super(MomentumTrainer, self).__init__(network, config, "MOMENTUM")

class FakeTrainer(GeneralNeuralTrainer):
    """
    Fake Trainer does nothing.
    """

    def _learning_updates(self):
        return []