def _token_set(s1, s2, partial=True, force_ascii=True, full_process=True):
    """Find all alphanumeric tokens in each string...
        - treat them as a set
        - construct two strings of the form:
            <sorted_intersection><sorted_remainder>
        - take ratios of those two strings
        - controls for unordered partial matches"""

    if not full_process and s1 == s2:
        return 100

    p1 = utils.full_process(s1, force_ascii=force_ascii) if full_process else s1
    p2 = utils.full_process(s2, force_ascii=force_ascii) if full_process else s2

    if not utils.validate_string(p1):
        return 0
    if not utils.validate_string(p2):
        return 0

    # pull tokens
    tokens1 = set(p1.split())
    tokens2 = set(p2.split())

    intersection = tokens1.intersection(tokens2)
    diff1to2 = tokens1.difference(tokens2)
    diff2to1 = tokens2.difference(tokens1)

    sorted_sect = " ".join(sorted(intersection))
    sorted_1to2 = " ".join(sorted(diff1to2))
    sorted_2to1 = " ".join(sorted(diff2to1))

    combined_1to2 = sorted_sect + " " + sorted_1to2
    combined_2to1 = sorted_sect + " " + sorted_2to1

    # strip
    sorted_sect = sorted_sect.strip()
    combined_1to2 = combined_1to2.strip()
    combined_2to1 = combined_2to1.strip()

    if partial:
        ratio_func = partial_ratio
    else:
        ratio_func = ratio

    pairwise = [
        ratio_func(sorted_sect, combined_1to2),
        ratio_func(sorted_sect, combined_2to1),
        ratio_func(combined_1to2, combined_2to1)
    ]
    return max(pairwise)
def _token_set ( s1 , s2 , partial = True , force_ascii = True , full_process = True ) : if not full_process and s1 == s2 : return 100 p1 = utils . full_process ( s1 , force_ascii = force_ascii ) if full_process else s1 p2 = utils . full_process ( s2 , force_ascii = force_ascii ) if full_process else s2 if not utils . validate_string ( p1 ) : return 0 if not utils . validate_string ( p2 ) : return 0 tokens1 = set ( p1 . split ( ) ) tokens2 = set ( p2 . split ( ) ) intersection = tokens1 . intersection ( tokens2 ) diff1to2 = tokens1 . difference ( tokens2 ) diff2to1 = tokens2 . difference ( tokens1 ) sorted_sect = " " . join ( sorted ( intersection ) ) sorted_1to2 = " " . join ( sorted ( diff1to2 ) ) sorted_2to1 = " " . join ( sorted ( diff2to1 ) ) combined_1to2 = sorted_sect + " " + sorted_1to2 combined_2to1 = sorted_sect + " " + sorted_2to1 sorted_sect = sorted_sect . strip ( ) combined_1to2 = combined_1to2 . strip ( ) combined_2to1 = combined_2to1 . strip ( ) if partial : ratio_func = partial_ratio else : ratio_func = ratio pairwise = [ ratio_func ( sorted_sect , combined_1to2 ) , ratio_func ( sorted_sect , combined_2to1 ) , ratio_func ( combined_1to2 , combined_2to1 ) ] return max ( pairwise )
Find all alphanumeric tokens in each string...
        - treat them as a set
        - construct two strings of the form:
            <sorted_intersection><sorted_remainder>
        - take ratios of those two strings
        - controls for unordered partial matches