<s> import fnmatch <EOL> import re <EOL> from basename_ranker import BasenameRanker <EOL> from trace_event import * <EOL> class DBIndexShard ( object ) : <EOL> def __init__ ( self , basenames ) : <EOL> basenames . sort ( ) <EOL> lower_basenames = set ( ) <EOL> for basename in basenames : <EOL> lower_basename = basename . lower ( ) <EOL> lower_basenames . add ( lower_basename ) <EOL> self . basenames_unsplit = ( "<STR_LIT:\n>" + "<STR_LIT:\n>" . join ( basenames ) + "<STR_LIT:\n>" ) . encode ( '<STR_LIT:utf8>' ) <EOL> self . lower_basenames_unsplit = ( "<STR_LIT:\n>" + "<STR_LIT:\n>" . join ( lower_basenames ) + "<STR_LIT:\n>" ) . encode ( '<STR_LIT:utf8>' ) <EOL> assert type ( self . lower_basenames_unsplit ) == str <EOL> self . _basename_ranker = BasenameRanker ( ) <EOL> wordstarts = { } <EOL> for basename in basenames : <EOL> start_letters = self . _basename_ranker . get_start_letters ( basename ) <EOL> if len ( start_letters ) <= <NUM_LIT:1> : <EOL> continue <EOL> lower_basename = basename . lower ( ) <EOL> for i in range ( len ( start_letters ) + <NUM_LIT:1> - <NUM_LIT:2> ) : <EOL> ws = '<STR_LIT>' . join ( start_letters [ <NUM_LIT:0> : <NUM_LIT:2> + i ] ) <EOL> if ws not in wordstarts : <EOL> wordstarts [ ws ] = [ ] <EOL> loss = len ( start_letters ) - ( <NUM_LIT:2> + i ) <EOL> wordstarts [ ws ] . append ( ( lower_basename , loss ) ) <EOL> self . basenames_by_wordstarts = { } <EOL> for ws , items in wordstarts . iteritems ( ) : <EOL> items . sort ( lambda x , y : cmp ( x [ <NUM_LIT:1> ] , y [ <NUM_LIT:1> ] ) ) <EOL> self . basenames_by_wordstarts [ ws ] = [ i [ <NUM_LIT:0> ] for i in items ] <EOL> @ traced <EOL> def search_basenames ( self , query ) : <EOL> """<STR_LIT>""" <EOL> lower_query = query . lower ( ) <EOL> lower_hits = set ( ) <EOL> max_hits_hint = <NUM_LIT> <EOL> trace_begin ( "<STR_LIT>" ) <EOL> self . add_all_matching ( lower_hits , query , self . get_exact_match_filter ( lower_query ) , max_hits_hint ) <EOL> trace_end ( "<STR_LIT>" ) <EOL> trace_begin ( "<STR_LIT>" ) <EOL> self . add_all_wordstarts_matching ( lower_hits , query , max_hits_hint ) <EOL> trace_end ( "<STR_LIT>" ) <EOL> trace_begin ( "<STR_LIT>" ) <EOL> self . add_all_matching ( lower_hits , query , self . get_substring_filter ( lower_query ) , max_hits_hint ) <EOL> trace_end ( "<STR_LIT>" ) <EOL> has_hq = False <EOL> for lower_hit in lower_hits : <EOL> rank = self . _basename_ranker . rank_query ( query , lower_hit ) <EOL> if rank > <NUM_LIT:2> : <EOL> has_hq = True <EOL> break <EOL> if not has_hq : <EOL> trace_begin ( "<STR_LIT>" ) <EOL> self . add_all_matching ( lower_hits , query , self . get_superfuzzy_filter ( lower_query ) , max_hits_hint ) <EOL> trace_end ( "<STR_LIT>" ) <EOL> return lower_hits , len ( lower_hits ) == max_hits_hint <EOL> def add_all_wordstarts_matching ( self , lower_hits , query , max_hits_hint ) : <EOL> lower_query = query . lower ( ) <EOL> if lower_query in self . basenames_by_wordstarts : <EOL> for basename in self . basenames_by_wordstarts [ lower_query ] : <EOL> lower_hits . add ( basename ) <EOL> if len ( lower_hits ) >= max_hits_hint : <EOL> return <EOL> def get_exact_match_filter ( self , query ) : <EOL> query = re . escape ( query . lower ( ) ) <EOL> flt = "<STR_LIT>" % query <EOL> return ( flt , False ) <EOL> def get_delimited_wordstart_filter ( self , query ) : <EOL> query = [ re . escape ( query [ i ] ) for i in range ( len ( query ) ) ] <EOL> tmp = [ ] <EOL> tmp . append ( "<STR_LIT>" % ( query [ <NUM_LIT:0> ] , query [ <NUM_LIT:0> ] ) ) <EOL> for i in range ( <NUM_LIT:1> , len ( query ) ) : <EOL> c = query [ i ] <EOL> tmp . append ( "<STR_LIT>" % query [ i ] ) <EOL> flt = "<STR_LIT>" % '<STR_LIT>' . join ( tmp ) <EOL> return ( flt , False ) <EOL> def get_camelcase_wordstart_filter ( self , query ) : <EOL> query = query . upper ( ) <EOL> query = [ re . escape ( query [ i ] ) for i in range ( len ( query ) ) ] <EOL> tmp = [ ] <EOL> tmp . append ( "<STR_LIT>" % ( query [ <NUM_LIT:0> ] , query [ <NUM_LIT:0> ] ) ) <EOL> for i in range ( <NUM_LIT:1> , len ( query ) ) : <EOL> tmp . append ( "<STR_LIT>" % query [ i ] ) <EOL> flt = "<STR_LIT>" % '<STR_LIT>' . join ( tmp ) <EOL> return ( flt , True ) <EOL> def get_substring_filter ( self , query ) : <EOL> query = re . escape ( query . lower ( ) ) <EOL> flt = "<STR_LIT>" % query <EOL> return ( flt , False ) <EOL> def get_superfuzzy_filter ( self , query ) : <EOL> tmp = [ ] <EOL> for i in range ( len ( query ) ) : <EOL> tmp . append ( re . escape ( query [ i ] ) ) <EOL> flt = "<STR_LIT>" % '<STR_LIT>' . join ( tmp ) <EOL> return ( flt , False ) <EOL> def add_all_matching ( self , lower_hits , query , flt_tuple , max_hits_hint ) : <EOL> """<STR_LIT>""" <EOL> flt , case_sensitive = flt_tuple <EOL> regex = re . compile ( flt ) <EOL> base = <NUM_LIT:0> <EOL> if not case_sensitive : <EOL> index = self . lower_basenames_unsplit <EOL> else : <EOL> index = self . basenames_unsplit <EOL> while True : <EOL> m = regex . search ( index , base ) <EOL> if m : <EOL> hit = m . group ( <NUM_LIT:0> ) [ <NUM_LIT:1> : - <NUM_LIT:1> ] <EOL> if hit . find ( '<STR_LIT:\n>' ) != - <NUM_LIT:1> : <EOL> raise Exception ( "<STR_LIT>" % ( flt , query , hit ) ) <EOL> if case_sensitive : <EOL> hit = hit . lower ( ) <EOL> lower_hits . add ( hit ) <EOL> base = m . end ( ) - <NUM_LIT:1> <EOL> if len ( lower_hits ) >= max_hits_hint : <EOL> truncated = True <EOL> break <EOL> else : <EOL> break </s>
