<s> '''<STR_LIT>''' <EOL> import normdb <EOL> import simstringdb <EOL> import sdistance <EOL> from datetime import datetime <EOL> from message import Messager <EOL> from normdb import string_norm_form <EOL> from document import real_directory <EOL> from projectconfig import ProjectConfiguration <EOL> DISPLAY_SEARCH_SCORES = False <EOL> MAX_SCORE = <NUM_LIT:1000> <EOL> MAX_DIFF_TO_BEST_SCORE = <NUM_LIT:200> <EOL> MAX_SEARCH_RESULT_NUMBER = <NUM_LIT:1000> <EOL> NORM_LOOKUP_DEBUG = True <EOL> REPORT_LOOKUP_TIMINGS = False <EOL> def _check_DB_version ( database ) : <EOL> pass <EOL> def _report_timings ( dbname , start , msg = None ) : <EOL> delta = datetime . now ( ) - start <EOL> strdelta = str ( delta ) . replace ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> queries = normdb . get_query_count ( dbname ) <EOL> normdb . reset_query_count ( dbname ) <EOL> Messager . info ( "<STR_LIT>" + str ( queries ) + "<STR_LIT>" + strdelta + <EOL> ( msg if msg is not None else "<STR_LIT>" ) ) <EOL> def _get_db_path ( database , collection ) : <EOL> if collection is None : <EOL> return None <EOL> else : <EOL> try : <EOL> conf_dir = real_directory ( collection ) <EOL> projectconf = ProjectConfiguration ( conf_dir ) <EOL> norm_conf = projectconf . get_normalization_config ( ) <EOL> for entry in norm_conf : <EOL> dbname , dbpath = entry [ <NUM_LIT:0> ] , entry [ <NUM_LIT:3> ] <EOL> if dbname == database : <EOL> return dbpath <EOL> Messager . warning ( '<STR_LIT>' + database + '<STR_LIT>' + <EOL> collection + '<STR_LIT>' ) <EOL> return None <EOL> except Exception : <EOL> Messager . warning ( '<STR_LIT>' + <EOL> collection + '<STR_LIT>' ) <EOL> return None <EOL> def norm_get_name ( database , key , collection = None ) : <EOL> if NORM_LOOKUP_DEBUG : <EOL> _check_DB_version ( database ) <EOL> if REPORT_LOOKUP_TIMINGS : <EOL> lookup_start = datetime . now ( ) <EOL> dbpath = _get_db_path ( database , collection ) <EOL> if dbpath is None : <EOL> dbpath = database <EOL> try : <EOL> data = normdb . data_by_id ( dbpath , key ) <EOL> except normdb . dbNotFoundError , e : <EOL> Messager . warning ( str ( e ) ) <EOL> data = None <EOL> if data is not None : <EOL> value = data [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] <EOL> else : <EOL> value = None <EOL> if REPORT_LOOKUP_TIMINGS : <EOL> _report_timings ( database , lookup_start ) <EOL> json_dic = { <EOL> '<STR_LIT>' : database , <EOL> '<STR_LIT:key>' : key , <EOL> '<STR_LIT:value>' : value <EOL> } <EOL> return json_dic <EOL> def norm_get_data ( database , key , collection = None ) : <EOL> if NORM_LOOKUP_DEBUG : <EOL> _check_DB_version ( database ) <EOL> if REPORT_LOOKUP_TIMINGS : <EOL> lookup_start = datetime . now ( ) <EOL> dbpath = _get_db_path ( database , collection ) <EOL> if dbpath is None : <EOL> dbpath = database <EOL> try : <EOL> data = normdb . data_by_id ( dbpath , key ) <EOL> except normdb . dbNotFoundError , e : <EOL> Messager . warning ( str ( e ) ) <EOL> data = None <EOL> if data is None : <EOL> Messager . warning ( "<STR_LIT>" + database + "<STR_LIT::>" + key ) <EOL> if REPORT_LOOKUP_TIMINGS : <EOL> _report_timings ( database , lookup_start ) <EOL> json_dic = { <EOL> '<STR_LIT>' : database , <EOL> '<STR_LIT:key>' : key , <EOL> '<STR_LIT:value>' : data <EOL> } <EOL> return json_dic <EOL> def _format_datas ( datas , scores = None , matched = None ) : <EOL> if scores is None : <EOL> scores = { } <EOL> if matched is None : <EOL> matched = { } <EOL> cropped = { } <EOL> for key in datas : <EOL> cropped [ key ] = datas [ key ] [ : <NUM_LIT:2> ] <EOL> datas = cropped <EOL> unique_labels = [ ] <EOL> seen_label = { } <EOL> for key in datas : <EOL> seen_label_for_key = { } <EOL> for i , group in enumerate ( datas [ key ] ) : <EOL> for label , value in group : <EOL> if label not in seen_label : <EOL> unique_labels . append ( ( i , label ) ) <EOL> seen_label [ label ] = True <EOL> if label in seen_label_for_key : <EOL> pass <EOL> seen_label_for_key [ label ] = True <EOL> unique_labels . sort ( lambda a , b : cmp ( a [ <NUM_LIT:0> ] , b [ <NUM_LIT:0> ] ) ) <EOL> unique_labels = [ a [ <NUM_LIT:1> ] for a in unique_labels ] <EOL> header = [ ( label , "<STR_LIT:string>" ) for label in [ "<STR_LIT>" ] + unique_labels ] <EOL> if DISPLAY_SEARCH_SCORES : <EOL> header += [ ( "<STR_LIT>" , "<STR_LIT:int>" ) ] <EOL> sorted_keys = sorted ( datas . keys ( ) , lambda a , b : cmp ( ( scores . get ( b , <NUM_LIT:0> ) , b ) , <EOL> ( scores . get ( a , <NUM_LIT:0> ) , a ) ) ) <EOL> items = [ ] <EOL> for key in sorted_keys : <EOL> data_dict = { } <EOL> for group in datas [ key ] : <EOL> for label , value in group : <EOL> if label not in data_dict or ( value in matched and <EOL> data_dict [ label ] not in matched ) : <EOL> data_dict [ label ] = value <EOL> item = [ str ( key ) ] <EOL> for label in unique_labels : <EOL> if label in data_dict : <EOL> item . append ( data_dict [ label ] ) <EOL> else : <EOL> item . append ( '<STR_LIT>' ) <EOL> if DISPLAY_SEARCH_SCORES : <EOL> item += [ str ( scores . get ( key ) ) ] <EOL> items . append ( item ) <EOL> return header , items <EOL> def _norm_filter_score ( score , best_score = MAX_SCORE ) : <EOL> return score < best_score - MAX_DIFF_TO_BEST_SCORE <EOL> def _norm_score ( substring , name , max_cost = <NUM_LIT> ) : <EOL> cache = _norm_score . __cache <EOL> if ( substring , name ) not in cache : <EOL> cost = sdistance . tsuruoka_local ( substring , name , max_cost = max_cost ) <EOL> score = MAX_SCORE - cost <EOL> cache [ ( substring , name ) ] = score <EOL> return cache [ ( substring , name ) ] <EOL> _norm_score . __cache = { } <EOL> def _norm_search_name_attr ( database , name , attr , <EOL> matched , score_by_id , score_by_str , <EOL> best_score = <NUM_LIT:0> , exactmatch = False , <EOL> threshold = simstringdb . DEFAULT_THRESHOLD ) : <EOL> if attr is not None : <EOL> utfattr = attr . encode ( '<STR_LIT>' ) <EOL> normattr = string_norm_form ( utfattr ) <EOL> if not simstringdb . ssdb_supstring_exists ( normattr , database , <NUM_LIT:1.0> ) : <EOL> return best_score <EOL> if exactmatch : <EOL> strs = [ name ] <EOL> ss_norm_score = { string_norm_form ( name ) : <NUM_LIT:1.0> } <EOL> else : <EOL> utfname = name . encode ( '<STR_LIT>' ) <EOL> normname = string_norm_form ( utfname ) <EOL> str_scores = simstringdb . ssdb_supstring_lookup ( normname , database , <EOL> threshold , True ) <EOL> strs = [ s [ <NUM_LIT:0> ] for s in str_scores ] <EOL> ss_norm_score = dict ( str_scores ) <EOL> if attr is None : <EOL> id_names = normdb . ids_by_names ( database , strs , False , True ) <EOL> else : <EOL> id_names = normdb . ids_by_names_attr ( database , strs , attr , False , True ) <EOL> id_name_scores = [ ( i , n , ss_norm_score [ string_norm_form ( n ) ] ) <EOL> for i , n in id_names ] <EOL> id_name_scores . sort ( lambda a , b : cmp ( b [ <NUM_LIT:2> ] , a [ <NUM_LIT:2> ] ) ) <EOL> id_names = [ ( i , n ) for i , n , s in id_name_scores ] <EOL> for i , n in id_names : <EOL> if n not in matched : <EOL> matched [ n ] = set ( ) <EOL> matched [ n ] . add ( i ) <EOL> max_cost = MAX_SCORE - best_score + MAX_DIFF_TO_BEST_SCORE + <NUM_LIT:1> <EOL> if ( name , n ) not in score_by_str : <EOL> score_by_str [ ( name , n ) ] = _norm_score ( string_norm_form ( name ) , string_norm_form ( n ) , max_cost ) <EOL> score = score_by_str [ ( name , n ) ] <EOL> best_score = max ( score , best_score ) <EOL> score_by_id [ i ] = max ( score_by_id . get ( i , - <NUM_LIT:1> ) , <EOL> score_by_str [ ( name , n ) ] ) <EOL> if len ( score_by_id ) > MAX_SEARCH_RESULT_NUMBER : <EOL> Messager . info ( '<STR_LIT>' % MAX_SEARCH_RESULT_NUMBER ) <EOL> break <EOL> return best_score <EOL> def _norm_search_impl ( database , name , collection = None , exactmatch = False ) : <EOL> if NORM_LOOKUP_DEBUG : <EOL> _check_DB_version ( database ) <EOL> if REPORT_LOOKUP_TIMINGS : <EOL> lookup_start = datetime . now ( ) <EOL> dbpath = _get_db_path ( database , collection ) <EOL> if dbpath is None : <EOL> dbpath = database <EOL> matched = { } <EOL> score_by_id = { } <EOL> score_by_str = { } <EOL> best_score = _norm_search_name_attr ( dbpath , name , None , <EOL> matched , score_by_id , score_by_str , <EOL> <NUM_LIT:0> , exactmatch ) <EOL> if best_score == <NUM_LIT:0> and len ( name . split ( ) ) == <NUM_LIT:1> : <EOL> best_score = _norm_search_name_attr ( dbpath , name , None , <EOL> matched , score_by_id , score_by_str , <EOL> <NUM_LIT:0> , exactmatch , <NUM_LIT:0.5> ) <EOL> if best_score < <NUM_LIT> and not exactmatch : <EOL> parts = name . split ( ) <EOL> for i in range ( len ( parts ) - <NUM_LIT:1> , <NUM_LIT:0> , - <NUM_LIT:1> ) : <EOL> if len ( score_by_id ) > MAX_SEARCH_RESULT_NUMBER : <EOL> break <EOL> start = '<STR_LIT:U+0020>' . join ( parts [ : i ] ) <EOL> end = '<STR_LIT:U+0020>' . join ( parts [ i : ] ) <EOL> best_score = _norm_search_name_attr ( dbpath , start , end , <EOL> matched , score_by_id , <EOL> score_by_str , <EOL> best_score , exactmatch ) <EOL> best_score = _norm_search_name_attr ( dbpath , end , start , <EOL> matched , score_by_id , <EOL> score_by_str , <EOL> best_score , exactmatch ) <EOL> ids = reduce ( set . union , matched . values ( ) , set ( ) ) <EOL> ids = set ( [ i for i in ids <EOL> if not _norm_filter_score ( score_by_id [ i ] , best_score ) ] ) <EOL> datas = normdb . datas_by_ids ( dbpath , ids ) <EOL> header , items = _format_datas ( datas , score_by_id , matched ) <EOL> if REPORT_LOOKUP_TIMINGS : <EOL> _report_timings ( database , lookup_start , <EOL> "<STR_LIT>" + str ( len ( items ) ) + "<STR_LIT>" ) <EOL> json_dic = { <EOL> '<STR_LIT>' : database , <EOL> '<STR_LIT>' : name , <EOL> '<STR_LIT>' : header , <EOL> '<STR_LIT>' : items , <EOL> } <EOL> return json_dic <EOL> def norm_search ( database , name , collection = None , exactmatch = False ) : <EOL> try : <EOL> return _norm_search_impl ( database , name , collection , exactmatch ) <EOL> except simstringdb . ssdbNotFoundError , e : <EOL> Messager . warning ( str ( e ) ) <EOL> return { <EOL> '<STR_LIT>' : database , <EOL> '<STR_LIT>' : name , <EOL> '<STR_LIT>' : [ ] , <EOL> '<STR_LIT>' : [ ] <EOL> } <EOL> def _test ( ) : <EOL> test_cases = { <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> } <EOL> } <EOL> overall_start = datetime . now ( ) <EOL> query_count , hit_count = <NUM_LIT:0> , <NUM_LIT:0> <EOL> misses = [ ] <EOL> for DB in test_cases : <EOL> for query in test_cases [ DB ] : <EOL> target = test_cases [ DB ] [ query ] <EOL> start = datetime . now ( ) <EOL> results = norm_search ( DB , query ) <EOL> delta = datetime . now ( ) - start <EOL> found = False <EOL> found_rank = - <NUM_LIT:1> <EOL> for rank , item in enumerate ( results [ '<STR_LIT>' ] ) : <EOL> id_ = item [ <NUM_LIT:0> ] <EOL> if id_ == target : <EOL> found = True <EOL> found_rank = rank + <NUM_LIT:1> <EOL> break <EOL> strdelta = str ( delta ) . replace ( '<STR_LIT>' , '<STR_LIT>' ) . replace ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> print "<STR_LIT>" % ( '<STR_LIT>' if found <EOL> else '<STR_LIT>' , <EOL> target , query , <EOL> found_rank , <EOL> len ( results [ '<STR_LIT>' ] ) , <EOL> strdelta ) <EOL> query_count += <NUM_LIT:1> <EOL> if found : <EOL> hit_count += <NUM_LIT:1> <EOL> else : <EOL> misses . append ( ( query , target ) ) <EOL> if len ( misses ) != <NUM_LIT:0> : <EOL> print <EOL> print "<STR_LIT>" <EOL> for query , target in misses : <EOL> print "<STR_LIT>" % ( target , query ) <EOL> delta = datetime . now ( ) - overall_start <EOL> strdelta = str ( delta ) . replace ( '<STR_LIT>' , '<STR_LIT>' ) . replace ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> print <EOL> print "<STR_LIT>" % ( hit_count , query_count , strdelta ) <EOL> def _profile_test ( ) : <EOL> import cProfile <EOL> import os . path <EOL> cProfile . run ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> _test ( ) </s>
