Construct training data for consumption by the ActiveLearning
    markPairs method from an already deduplicated dataset.

    Arguments :
    data          -- Dictionary of records, where the keys are record_ids and
                     the values are dictionaries with the keys being
                     field names

    common_key    -- The name of the record field that uniquely identifies
                     a match

    training_size -- the rough limit of the number of training examples,
                     defaults to 50000

    Warning:

    Every match must be identified by the sharing of a common key.
    This function assumes that if two records do not share a common key
    then they are distinct records.

def trainingDataDedupe(data, common_key, training_size=50000):  # pragma: nocover
    '''
    Construct training data for consumption by the ActiveLearning
    markPairs method from an already deduplicated dataset.

    Arguments :
    data          -- Dictionary of records, where the keys are record_ids and
                     the values are dictionaries with the keys being
                     field names

    common_key    -- The name of the record field that uniquely identifies
                     a match

    training_size -- the rough limit of the number of training examples,
                     defaults to 50000

    Warning:

    Every match must be identified by the sharing of a common key.
    This function assumes that if two records do not share a common key
    then they are distinct records.
    '''

    identified_records = collections.defaultdict(list)
    matched_pairs = set()
    distinct_pairs = set()
    unique_record_ids = set()

    # a list of record_ids associated with each common_key
    for record_id, record in data.items():
        unique_record_ids.add(record_id)
        identified_records[record[common_key]].append(record_id)

    # all combinations of matched_pairs from each common_key group
    for record_ids in identified_records.values():
        if len(record_ids) > 1:
            matched_pairs.update(itertools.combinations(sorted(record_ids), 2))

    # calculate indices using dedupe.core.randomPairs to avoid
    # the memory cost of enumerating all possible pairs
    unique_record_ids = list(unique_record_ids)
    pair_indices = randomPairs(len(unique_record_ids), training_size)
    distinct_pairs = set()
    for i, j in pair_indices:
        distinct_pairs.add((unique_record_ids[i],
                            unique_record_ids[j]))

    distinct_pairs -= matched_pairs

    matched_records = [(data[key_1], data[key_2])
                       for key_1, key_2 in matched_pairs]

    distinct_records = [(data[key_1], data[key_2])
                        for key_1, key_2 in distinct_pairs]

    training_pairs = {'match': matched_records,
                      'distinct': distinct_records}

    return training_pairs