def build_dict_from_table(table, key_attr_index, join_attr_index,
                          remove_null=True):
    table_dict = {}
    for row in table.itertuples(index=False):
        if remove_null and pd.isnull(row[join_attr_index]):
            continue
        table_dict[row[key_attr_index]] = tuple(row)
    return table_dict def find_output_attribute_indices(original_columns, output_attributes):
    output_attribute_indices = []
    if output_attributes is not None:
        for attr in output_attributes:
            output_attribute_indices.append(original_columns.index(attr))
    return output_attribute_indices def get_attrs_to_project(out_attrs, key_attr, join_attr):
    # this method assumes key_attr has already been removed from
    # out_attrs, if present.
    proj_attrs = [key_attr, join_attr]

    if out_attrs is not None:
        for attr in out_attrs:
            if attr != join_attr:
                proj_attrs.append(attr)

    return proj_attrs def get_num_processes_to_launch(n_jobs):        
    # determine number of processes to launch parallely
    num_procs = n_jobs
    if n_jobs < 0:
        num_cpus = multiprocessing.cpu_count()
        num_procs = num_cpus + 1 + n_jobs
    return max(num_procs, 1) def get_output_header_from_tables(l_key_attr, r_key_attr,
                                  l_out_attrs, r_out_attrs,
                                  l_out_prefix, r_out_prefix):
    output_header = []

    output_header.append(l_out_prefix + l_key_attr)

    output_header.append(r_out_prefix + r_key_attr)

    if l_out_attrs:
        for l_attr in l_out_attrs:
            output_header.append(l_out_prefix + l_attr)

    if r_out_attrs:
        for r_attr in r_out_attrs:
            output_header.append(r_out_prefix + r_attr)

    return output_header def get_output_row_from_tables(l_row, r_row,
                               l_key_attr_index, r_key_attr_index, 
                               l_out_attrs_indices=None,
                               r_out_attrs_indices=None):
    output_row = []
    
    # add ltable id attr
    output_row.append(l_row[l_key_attr_index])

    # add rtable id attr
    output_row.append(r_row[r_key_attr_index])

    # add ltable output attributes
    if l_out_attrs_indices:
        for l_attr_index in l_out_attrs_indices:
            output_row.append(l_row[l_attr_index])

    # add rtable output attributes
    if r_out_attrs_indices:
        for r_attr_index in r_out_attrs_indices:
            output_row.append(r_row[r_attr_index])

    return output_row def remove_redundant_attrs(out_attrs, key_attr):
    # this method removes key_attr from out_attrs, if present.
    # further it removes redundant attributes in out_attrs, 
    # but preserves the order of attributes.
    if out_attrs is None:
        return out_attrs

    uniq_attrs = []
    seen_attrs = {}
    for attr in out_attrs:
        if attr == key_attr or seen_attrs.get(attr) is not None:
            continue
        uniq_attrs.append(attr)
        seen_attrs[attr] = True

    return uniq_attrs def split_table(table, num_splits):
    splits = []
    split_size = 1.0/num_splits*len(table)
    for i in xrange(num_splits):
        splits.append(table[int(round(i*split_size)):
                            int(round((i+1)*split_size))])
    return splits COMP_OP_MAP = {'>=': operator.ge,
               '>': operator.gt,
               '<=': operator.le,
               '<': operator.lt,
               '=': operator.eq,
               '!=': operator.ne} def pickle_instance_method(method):
    """Pickle class instance method."""
    func_name = method.__func__.__name__
    obj = method.__self__
    cls = method.__self__.__class__
    return unpickle_instance_method, (func_name, obj, cls) def unpickle_instance_method(func_name, obj, cls):
    """Unpickle class instance method."""
    for cls in cls.mro():
        try:
            func = cls.__dict__[func_name]
        except KeyError:
            pass
        else:
            break
    return func.__get__(obj, cls) def validate_attr(attr, table_cols, attr_label, table_label):
    """Check if the attribute exists in the table."""
    if attr not in table_cols:
        raise AssertionError(attr_label + ' \'' + attr + '\' not found in ' + \
                             table_label) 
    return True def validate_comp_op(comp_op):
    """Check if the comparison operator is valid."""
    if comp_op not in COMP_OP_MAP.keys():
        raise AssertionError('Comparison operator not supported. ' + \
            'Supported comparison operators are >=, >, <=, <, = and !=.')         def validate_key_attr(key_attr, table, table_label):
    """Check if the attribute is a valid key attribute."""
    unique_flag = len(table[key_attr].unique()) == len(table)
    nan_flag = sum(table[key_attr].isnull()) == 0 
    if not (unique_flag and nan_flag):
        raise AssertionError('\'' + key_attr + '\' is not a key attribute ' + \
                             'in ' + table_label)
    return True def validate_input_table(table, table_label):
    """Check if the input table is a dataframe."""
    if not isinstance(table, pd.DataFrame):
        raise TypeError(table_label + ' is not a dataframe')
    return True def validate_tokenizer(tokenizer):
    """Check if the input tokenizer is a valid tokenizer."""
    if not isinstance(tokenizer, Tokenizer):
        raise TypeError('Invalid tokenizer provided as input')
    return True def validate_output_attrs(l_out_attrs, l_columns, r_out_attrs, r_columns):
    """Check if the output attributes exist in the original tables."""
    if l_out_attrs:
        for attr in l_out_attrs:
            if attr not in l_columns:
                raise AssertionError('output attribute \'' + attr + \
                                     '\' not found in left table')

    if r_out_attrs:
        for attr in r_out_attrs:
            if attr not in r_columns:
                raise AssertionError('output attribute \'' + attr + \
                                     '\' not found in right table')
    return True
import operator
import types
import numpy as np
import pandas as pd
import pyprind
from joblib import delayed, Parallel
from six.moves import copyreg
from py_stringsimjoin.utils.generic_helper import build_dict_from_table, \
    find_output_attribute_indices, get_attrs_to_project, \
    get_num_processes_to_launch, get_output_header_from_tables, \
    get_output_row_from_tables, remove_redundant_attrs, split_table, COMP_OP_MAP
from py_stringsimjoin.utils.pickle import pickle_instance_method, \
                                          unpickle_instance_method
from py_stringsimjoin.utils.validation import validate_attr, \
    validate_comp_op, validate_key_attr, validate_input_table, \
    validate_tokenizer, validate_output_attrs
                  l_out_attrs=None, r_out_attrs=None,
                  l_out_prefix='l_', r_out_prefix='r_',
                  out_sim_score=True, n_jobs=1, show_progress=True):
    """Find matching string pairs from the candidate set (typically produced by
    applying a filter to two tables) by applying a matcher of form 
    (sim_function comp_op threshold).

    Specifically, this method computes the input similarity function on string 
    pairs in the candidate set and checks if the resulting score satisfies the 
    input threshold (depending on the comparison operator).

    Args:
        candset (DataFrame): input candidate set.

        candset_l_key_attr (string): attribute in candidate set which is a key 
            in left table.

        candset_r_key_attr (string): attribute in candidate set which is a key 
            in right table.

        ltable (DataFrame): left input table.

        rtable (DataFrame): right input table.

        l_key_attr (string): key attribute in left table.

        r_key_attr (string): key attribute in right table.

        l_match_attr (string): attribute in left table on which the matcher 
            should be applied.

        r_match_attr (string): attribute in right table on which the matcher
            should be applied.

        tokenizer (Tokenizer): tokenizer to be used to tokenize the
            match attributes. If set to None, the matcher is applied directly
            on the match attributes.

        sim_function (function): matcher function to be applied.

        threshold (float): threshold to be satisfied.

        comp_op (string): comparison operator. Supported values are '>=', '>', '
            <=', '<', '=' and '!=' (defaults to '>=').

        allow_missing (boolean): flag to indicate whether tuple pairs with 
            missing value in at least one of the match attributes should be 
            included in the output (defaults to False). 

        l_out_attrs (list): list of attribute names from the left table to be 
            included in the output table (defaults to None).

        r_out_attrs (list): list of attribute names from the right table to be 
            included in the output table (defaults to None).

        l_out_prefix (string): prefix to be used for the attribute names coming 
            from the left table, in the output table (defaults to 'l\_').

        r_out_prefix (string): prefix to be used for the attribute names coming 
            from the right table, in the output table (defaults to 'r\_').

        out_sim_score (boolean): flag to indicate whether similarity score 
            should be included in the output table (defaults to True). Setting
            this flag to True will add a column named '_sim_score' in the 
            output table. This column will contain the similarity scores for the
            tuple pairs in the output. 

        n_jobs (int): number of parallel jobs to use for the computation        
            (defaults to 1). If -1 is given, all CPUs are used. If 1 is given,  
            no parallel computing code is used at all, which is useful for      
            debugging. For n_jobs below -1, (n_cpus + 1 + n_jobs) are used      
            (where n_cpus is the total number of CPUs in the machine). Thus for 
            n_jobs = -2, all CPUs but one are used. If (n_cpus + 1 + n_jobs)    
            becomes less than 1, then no parallel computing code will be used   
            (i.e., equivalent to the default). 

        show_progress (boolean): flag to indicate whether task progress should 
            be displayed to the user (defaults to True).

    Returns:
        An output table containing tuple pairs from the candidate set that 
        survive the matcher (DataFrame).
    """

    # check if the input candset is a dataframe
    validate_input_table(candset, 'candset')

    # check if the candset key attributes exist
    validate_attr(candset_l_key_attr, candset.columns,
                  'left key attribute', 'candset')
    validate_attr(candset_r_key_attr, candset.columns,
                  'right key attribute', 'candset')

    # check if the input tables are dataframes
    validate_input_table(ltable, 'left table')
    validate_input_table(rtable, 'right table')

    # check if the key attributes and join attributes exist
    validate_attr(l_key_attr, ltable.columns,
                  'key attribute', 'left table')
    validate_attr(r_key_attr, rtable.columns,
                  'key attribute', 'right table')
    validate_attr(l_match_attr, ltable.columns,
                  'match attribute', 'left table')
    validate_attr(r_match_attr, rtable.columns,
                  'match attribute', 'right table')

    # check if the output attributes exist
    validate_output_attrs(l_out_attrs, ltable.columns,
                          r_out_attrs, rtable.columns)

    # check if the input tokenizer is valid, if it is not None
    if tokenizer is not None:
        validate_tokenizer(tokenizer)

    # check if the comparison operator is valid
    validate_comp_op(comp_op)

    # check if the key attributes are unique and do not contain missing values
    validate_key_attr(l_key_attr, ltable, 'left table')

    validate_key_attr(r_key_attr, rtable, 'right table')