def follow_probs_reducer(self, start_word_category, follow_word_counts):
        start, category = start_word_category
        follow_counts = {}

        for follow_word, count in follow_word_counts:
            follow_counts[follow_word] = follow_counts.get(follow_word, 0) + count

        total_transitions = float(sum(follow_counts.itervalues()))

        include_word = lambda count: count > MINIMUM_PAIR_COUNT and count / total_transitions > MINIMUM_FOLLOW_PERCENTAGE
        thresholded_follow_counts = dict((word, count) for word, count in follow_counts.iteritems() if include_word(count))

        # filter out transitions where the transition has either
        # occurred a minimum number of times, or does not make up a
        # minimum percentage of outgoing transitions.
        if not thresholded_follow_counts:
            return

        # put a small weight on <end>, which means 'end of review'.
        thresholded_follow_counts['<end>'] = thresholded_follow_counts.get('<end>', 0.0) 
        thresholded_follow_counts['<end>'] += END_OF_REVIEW_RATE * float(sum(thresholded_follow_counts.itervalues()))

        # re-normalize the remaining transition weights.
        new_total = float(sum(thresholded_follow_counts.itervalues()))
        percentages = dict((follow, count / new_total) for follow, count in thresholded_follow_counts.iteritems())

        yield (category, start), percentages
Given a start word and a category, find the distribution
dataset/ETHPy150Open Yelp/dataset-examples/review_autopilot/autopilot.py/ReviewAutoPilot.follow_probs_reducer