<s> from fuzzywuzzy import fuzz <EOL> import jellyfish <EOL> import unicodedata , pickle <EOL> from reconciliation . utils . matchlength import longest_match <EOL> from django . db . models import Q <EOL> from ftpdata . models import Candidate <EOL> from nameparser import HumanName <EOL> from datetime import date <EOL> from operator import itemgetter <EOL> from reconciliation . utils . candidate_aliases import candidate_hash <EOL> from django . conf import settings <EOL> from reconciliation . utils . nicknames import nicknamedict <EOL> import logging <EOL> log = logging . getLogger ( '<STR_LIT>' ) <EOL> CHECK_FOR_NAME_REVERSALS = getattr ( settings , '<STR_LIT>' ) <EOL> default_cycle = '<STR_LIT>' <EOL> debug = True <EOL> starts_with_blocklength = <NUM_LIT:5> <EOL> def standardize_name_from_dict ( candidate ) : <EOL> district = "<STR_LIT>" <EOL> if candidate [ '<STR_LIT>' ] != '<STR_LIT>' : <EOL> district = "<STR_LIT>" % candidate [ '<STR_LIT>' ] <EOL> return "<STR_LIT>" % ( candidate [ '<STR_LIT>' ] , candidate [ '<STR_LIT>' ] , candidate [ '<STR_LIT>' ] , candidate [ '<STR_LIT>' ] , district ) <EOL> def block_by_startswith ( name , numchars , state = None , office = None , cycle = None ) : <EOL> namestart = name [ : numchars ] <EOL> if debug : <EOL> log . debug ( "<STR_LIT>" % ( name , state , office , cycle ) ) <EOL> print ( "<STR_LIT>" % ( name , state , office , cycle ) ) <EOL> matches = Candidate . objects . filter ( cand_name__istartswith = namestart ) <EOL> president_flag = False <EOL> if office : <EOL> if office . upper ( ) [ : <NUM_LIT:1> ] == '<STR_LIT:H>' : <EOL> matches = matches . filter ( cand_office = '<STR_LIT:H>' ) <EOL> elif office . upper ( ) [ : <NUM_LIT:1> ] == '<STR_LIT:S>' : <EOL> matches = matches . filter ( cand_office = '<STR_LIT:S>' ) <EOL> elif office . upper ( ) [ : <NUM_LIT:1> ] == '<STR_LIT:P>' : <EOL> matches = matches . filter ( cand_office = '<STR_LIT:P>' ) <EOL> president_flag = True <EOL> if state and not president_flag : <EOL> state = state . strip ( ) . upper ( ) <EOL> matches = matches . filter ( cand_office_st = state ) <EOL> if cycle and len ( str ( cycle ) ) > <NUM_LIT:3> : <EOL> matches = matches . filter ( cycle = cycle ) <EOL> matches = matches . order_by ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> match_values = matches . values ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) . distinct ( ) <EOL> return match_values <EOL> def simple_clean ( string ) : <EOL> try : <EOL> string = unicodedata . normalize ( '<STR_LIT>' , string ) . encode ( '<STR_LIT:ascii>' , '<STR_LIT:ignore>' ) <EOL> except TypeError : <EOL> pass <EOL> return string . strip ( ) . lower ( ) <EOL> def compute_scores ( array ) : <EOL> sorted_array = sorted ( array ) <EOL> truncated = sorted_array [ <NUM_LIT:1> : ] <EOL> avg = sum ( truncated ) / float ( len ( truncated ) ) <EOL> return avg <EOL> def unnickname ( firstname ) : <EOL> firstname = simple_clean ( firstname ) <EOL> try : <EOL> firstname = nicknamedict [ firstname ] <EOL> except KeyError : <EOL> pass <EOL> return firstname <EOL> def hash_lookup ( name , state = None , office = None , cycle = None ) : <EOL> result_array = [ ] <EOL> print "<STR_LIT>" % ( name , cycle , state , office ) <EOL> hashname = str ( name ) . upper ( ) . strip ( ) . strip ( '<STR_LIT:">' ) <EOL> print "<STR_LIT>" % ( hashname ) <EOL> if cycle and len ( str ( cycle ) ) > <NUM_LIT:3> : <EOL> hash_lookup_cycle = str ( cycle ) <EOL> else : <EOL> hash_lookup_cycle = default_cycle <EOL> if hash_lookup_cycle == default_cycle : <EOL> try : <EOL> found_candidate = candidate_hash [ hash_lookup_cycle ] [ hashname ] <EOL> except KeyError : <EOL> return None <EOL> if found_candidate : <EOL> valid_candidate = True <EOL> if office and len ( office ) > <NUM_LIT:0> : <EOL> if office . upper ( ) [ : <NUM_LIT:1> ] != found_candidate [ '<STR_LIT>' ] . upper ( ) [ : <NUM_LIT:1> ] : <EOL> valid_candidate = False <EOL> if state and len ( state ) > <NUM_LIT:1> : <EOL> if state . upper ( ) != found_candidate [ '<STR_LIT>' ] : <EOL> if office and office . upper ( ) [ : <NUM_LIT:1> ] == '<STR_LIT:P>' : <EOL> pass <EOL> else : <EOL> valid_candidate = False <EOL> if valid_candidate : <EOL> name_standardized = standardize_name_from_dict ( found_candidate ) <EOL> result_array . append ( { '<STR_LIT:name>' : name_standardized , '<STR_LIT:id>' : found_candidate [ '<STR_LIT>' ] , '<STR_LIT>' : <NUM_LIT:1> , '<STR_LIT:type>' : [ ] , '<STR_LIT>' : True } ) <EOL> return result_array <EOL> return None <EOL> def match_by_name ( name , state = None , office = None , cycle = None , reverse_name_order = False ) : <EOL> result_array = [ ] <EOL> name1 = HumanName ( name ) <EOL> name1_standardized = None <EOL> blocking_name = None <EOL> if reverse_name_order : <EOL> print "<STR_LIT>" <EOL> blocking_name = simple_clean ( name1 . first ) <EOL> name1_standardized = simple_clean ( name1 . first ) + "<STR_LIT:U+0020>" + unnickname ( name1 . last ) <EOL> else : <EOL> name1_standardized = simple_clean ( name1 . last ) + "<STR_LIT:U+0020>" + unnickname ( name1 . first ) <EOL> blocking_name = simple_clean ( name1 . last ) <EOL> if not blocking_name : <EOL> blocking_name = simple_clean ( name ) <EOL> possible_matches = block_by_startswith ( blocking_name , starts_with_blocklength , state , office , cycle ) <EOL> for match in possible_matches : <EOL> name2_name = HumanName ( match [ '<STR_LIT>' ] ) <EOL> name2 = simple_clean ( name2_name . last ) + "<STR_LIT:U+0020>" + unnickname ( name2_name . first ) <EOL> text1 = name1_standardized <EOL> text2 = name2 <EOL> ratio = <NUM_LIT:1> / <NUM_LIT> * fuzz . ratio ( text1 , text2 ) <EOL> partial_ratio = <NUM_LIT:1> / <NUM_LIT> * fuzz . partial_ratio ( text1 , text2 ) <EOL> token_sort_ratio = <NUM_LIT:1> / <NUM_LIT> * fuzz . token_sort_ratio ( text1 , text2 ) <EOL> token_set_ratio = <NUM_LIT:1> / <NUM_LIT> * fuzz . token_set_ratio ( text1 , text2 ) <EOL> avg_len = <NUM_LIT:1> / <NUM_LIT:2> * len ( text1 ) + len ( text2 ) <EOL> min_len = min ( len ( text1 ) , len ( text2 ) ) <EOL> l_ratio = <NUM_LIT:0> <EOL> try : <EOL> l_distance = jellyfish . levenshtein_distance ( text1 , text2 ) <EOL> l_ratio = <NUM_LIT:1.0> - ( ( <NUM_LIT:0.0> + l_distance ) / ( <NUM_LIT:0.0> + avg_len ) ) <EOL> except UnicodeEncodeError : <EOL> pass <EOL> long_match = longest_match ( text1 , text2 ) <EOL> lng_ratio = ( <NUM_LIT:0.0> + long_match ) / ( <NUM_LIT:0.0> + min_len ) <EOL> score = <NUM_LIT:0> <EOL> if ( ratio > <NUM_LIT> or partial_ratio > <NUM_LIT> or l_ratio > <NUM_LIT> or lng_ratio > <NUM_LIT> ) : <EOL> score = compute_scores ( [ ratio , partial_ratio , l_ratio , lng_ratio ] ) <EOL> if debug : <EOL> log . debug ( "<STR_LIT>" % ( match [ '<STR_LIT>' ] , match [ '<STR_LIT>' ] , name , score , ratio , partial_ratio , token_sort_ratio , token_set_ratio , l_ratio , lng_ratio ) ) <EOL> if ( score > <NUM_LIT> ) : <EOL> name_standardized = standardize_name_from_dict ( match ) <EOL> result_array . append ( { '<STR_LIT:name>' : name_standardized , '<STR_LIT:id>' : match [ '<STR_LIT>' ] , '<STR_LIT>' : score , '<STR_LIT:type>' : [ ] , '<STR_LIT>' : False } ) <EOL> if debug : <EOL> log . debug ( "<STR_LIT>" % name_standardized ) <EOL> if debug and len ( result_array ) == <NUM_LIT:0> : <EOL> log . debug ( "<STR_LIT>" % ( name , name1_standardized ) ) <EOL> if ( len ( result_array ) == <NUM_LIT:1> ) : <EOL> if result_array [ <NUM_LIT:0> ] [ '<STR_LIT>' ] > <NUM_LIT> : <EOL> result_array [ <NUM_LIT:0> ] [ '<STR_LIT>' ] = True <EOL> return result_array <EOL> def run_fec_query ( name , state = None , office = None , cycle = None , fuzzy = True ) : <EOL> if debug : <EOL> print "<STR_LIT>" % ( state , office , cycle ) <EOL> result = hash_lookup ( name , state , office , cycle ) <EOL> if result : <EOL> return result <EOL> if not fuzzy : <EOL> return [ ] <EOL> if ( len ( name ) < <NUM_LIT:4> ) : <EOL> return [ ] <EOL> result_array = match_by_name ( name , state = state , office = office , cycle = cycle , reverse_name_order = False ) <EOL> if CHECK_FOR_NAME_REVERSALS and len ( result_array ) == <NUM_LIT:0> : <EOL> result_array = match_by_name ( name , state = state , office = office , cycle = cycle , reverse_name_order = True ) <EOL> result_array = sorted ( result_array , key = itemgetter ( '<STR_LIT>' ) , reverse = True ) <EOL> return result_array </s>
