def dedupe(contains_dupes, threshold=70, scorer=fuzz.token_set_ratio):
 extractor = []
 for item in contains_dupes:
  matches = extract(item, contains_dupes, limit=None, scorer=scorer)
  filtered = [x for x in matches if (x[1] > threshold)]
  if (len(filtered) == 1):
   extractor.append(filtered[0][0])
  else:
   filtered = sorted(filtered, key=(lambda x: x[0]))
   filter_sort = sorted(filtered, key=(lambda x: len(x[0])), reverse=True)
   extractor.append(filter_sort[0][0])
 keys = {}
 for e in extractor:
  keys[e] = 1
 extractor = keys.keys()
 if (len(extractor) == len(contains_dupes)):
  return contains_dupes
 else:
  return extractor

def dedupe ( contains_dupes , threshold = 70 , scorer = fuzz . token_set_ratio ) : extractor = [ ] for item in contains_dupes : matches = extract ( item , contains_dupes , limit = None , scorer = scorer ) filtered = [ x for x in matches if ( x [ 1 ] > threshold ) ] if ( len ( filtered ) == 1 ) : extractor . append ( filtered [ 0 ] [ 0 ] ) else : filtered = sorted ( filtered , key = ( lambda x : x [ 0 ] ) ) filter_sort = sorted ( filtered , key = ( lambda x : len ( x [ 0 ] ) ) , reverse = True ) extractor . append ( filter_sort [ 0 ] [ 0 ] ) keys = { } for e in extractor : keys [ e ] = 1 extractor = keys . keys ( ) if ( len ( extractor ) == len ( contains_dupes ) ) : return contains_dupes else : return extractor
this convenience function takes a list of strings containing duplicates and uses fuzzy matching to identify and remove duplicates .