<filename>csverve/extras.py
from typing import List, Set, Dict, Tuple, Optional, Union
import pandas as pd # type: ignore
import numpy as np # type: ignore
import collections


def union_categories(dfs: List[pd.DataFrame], cat_cols: Union[Set[str], None] = None) -> None:
    """
    Recreate specified categoricals on the union of categories inplace.

    @param dfs: Llist of pandas dataframes to unify categoricals in-place.
    @param cat_cols: columns to unify categoricals, default None for any categorical.
    @return:
    """

    # Infer all categorical columns if not given
    if cat_cols is None:
        cat_cols = set()
        for df in dfs:
            for col in df:
                if df[col].dtype.name == 'category':
                    cat_cols.add(col)

    # Get a list of categories for each column
    col_categories = collections.defaultdict(set)
    for col in cat_cols:
        for df in dfs:
            if col in df:
                col_categories[col].update(df[col].values)

    # Remove None and nan as they cannot be in the list of categories
    for col in col_categories:
        col_categories[col] = col_categories[col] - set([None, np.nan])

    # Create a pandas index for each set of categories
    for col, categories in col_categories.items():
        col_categories[col] = pd.Index(categories)

    # Set all categorical columns as having teh same set of categories
    for col in cat_cols:
        for df in dfs:
            if col in df:
                df[col] = df[col].astype('category')
                df[col] = df[col].cat.set_categories(col_categories[col])


def concatenate_with_categories(dfs: List[pd.DataFrame], **kwargs) -> pd.DataFrame:
    """
    Concatenate dataframes retaining categorical columns.

    @param dfs: List of pandas dataframes.
    @param kwargs: Additional arguments (currently not in use).
    @return: Concatenated dataframes with categorial columns retained.
    """

    # Infer all categorical columns
    cat_cols = set()
    for df in dfs:
        for col in df:
            if df[col].dtype.name == 'category':
                cat_cols.add(col)

    # Get a list of categories for each column
    col_categories = collections.defaultdict(set)
    for df in dfs:
        for col in cat_cols:
            col_categories[col].update(df[col].values)

    # Remove None and nan as they cannot be in the list of categories
    for col in col_categories:
        col_categories[col] = col_categories[col] - set([None, np.nan])

    # Create a pandas index for each set of categories
    for col, categories in col_categories.items():
        col_categories[col] = pd.Index(categories)

    # Set all categorical columns as having teh same set of categories
    for df in dfs:
        for col in cat_cols:
            df[col] = df[col].astype('category')
            df[col] = df[col].cat.set_categories(col_categories[col])

    return pd.concat(dfs, **kwargs)

<filename>csverve/extras.py
from typing import List, Set, Dict, Tuple, Optional, Union
import pandas as pd # type: ignore
import numpy as np # type: ignore
import collections


def union_categories(dfs: List[pd.DataFrame], cat_cols: Union[Set[str], None] = None) -> None:
    """
    Recreate specified categoricals on the union of categories inplace.

    @param dfs: Llist of pandas dataframes to unify categoricals in-place.
    @param cat_cols: columns to unify categoricals, default None for any categorical.
    @return:
    """

    # Infer all categorical columns if not given
    if cat_cols is None:
        cat_cols = set()
        for df in dfs:
            for col in df:
                if df[col].dtype.name == 'category':
                    cat_cols.add(col)

    # Get a list of categories for each column
    col_categories = collections.defaultdict(set)
    for col in cat_cols:
        for df in dfs:
            if col in df:
                col_categories[col].update(df[col].values)

    # Remove None and nan as they cannot be in the list of categories
    for col in col_categories:
        col_categories[col] = col_categories[col] - set([None, np.nan])

    # Create a pandas index for each set of categories
    for col, categories in col_categories.items():
        col_categories[col] = pd.Index(categories)

    # Set all categorical columns as having teh same set of categories
    for col in cat_cols:
        for df in dfs:
            if col in df:
                df[col] = df[col].astype('category')
                df[col] = df[col].cat.set_categories(col_categories[col])


def concatenate_with_categories(dfs: List[pd.DataFrame], **kwargs) -> pd.DataFrame:
    """
    Concatenate dataframes retaining categorical columns.

    @param dfs: List of pandas dataframes.
    @param kwargs: Additional arguments (currently not in use).
    @return: Concatenated dataframes with categorial columns retained.
    """

    # Infer all categorical columns
    cat_cols = set()
    for df in dfs:
        for col in df:
            if df[col].dtype.name == 'category':
                cat_cols.add(col)

    # Get a list of categories for each column
    col_categories = collections.defaultdict(set)
    for df in dfs:
        for col in cat_cols:
            col_categories[col].update(df[col].values)

    # Remove None and nan as they cannot be in the list of categories
    for col in col_categories:
        col_categories[col] = col_categories[col] - set([None, np.nan])

    # Create a pandas index for each set of categories
    for col, categories in col_categories.items():
        col_categories[col] = pd.Index(categories)

    # Set all categorical columns as having teh same set of categories
    for df in dfs:
        for col in cat_cols:
            df[col] = df[col].astype('category')
            df[col] = df[col].cat.set_categories(col_categories[col])

    return pd.concat(dfs, **kwargs)

# type: ignore # type: ignore Recreate specified categoricals on the union of categories inplace.      @param dfs: Llist of pandas dataframes to unify categoricals in-place.     @param cat_cols: columns to unify categoricals, default None for any categorical.     @return: # Infer all categorical columns if not given # Get a list of categories for each column # Remove None and nan as they cannot be in the list of categories # Create a pandas index for each set of categories # Set all categorical columns as having teh same set of categories Concatenate dataframes retaining categorical columns.      @param dfs: List of pandas dataframes.     @param kwargs: Additional arguments (currently not in use).     @return: Concatenated dataframes with categorial columns retained. # Infer all categorical columns # Get a list of categories for each column # Remove None and nan as they cannot be in the list of categories # Create a pandas index for each set of categories # Set all categorical columns as having teh same set of categories