<s> import time <EOL> import logging <EOL> import chromaprint <EOL> from contextlib import closing <EOL> from sqlalchemy import sql <EOL> from acoustid import tables as schema , const <EOL> from acoustid . indexclient import IndexClientError <EOL> logger = logging . getLogger ( __name__ ) <EOL> FINGERPRINT_VERSION = <NUM_LIT:1> <EOL> PARTS = ( ( <NUM_LIT:1> , <NUM_LIT:20> ) , ( <NUM_LIT> , <NUM_LIT:100> ) ) <EOL> PART_SEARCH_SQL = """<STR_LIT>""" <EOL> def decode_fingerprint ( fingerprint_string ) : <EOL> """<STR_LIT>""" <EOL> fingerprint , version = chromaprint . decode_fingerprint ( fingerprint_string ) <EOL> if version == FINGERPRINT_VERSION : <EOL> return fingerprint <EOL> def lookup_fingerprint ( conn , fp , length , good_enough_score , min_score , fast = False , max_offset = <NUM_LIT:0> ) : <EOL> """<STR_LIT>""" <EOL> matched = [ ] <EOL> best_score = <NUM_LIT:0.0> <EOL> for part_start , part_length in PARTS : <EOL> params = dict ( fp = fp , length = length , part_start = part_start , <EOL> part_length = part_length , max_length_diff = const . FINGERPRINT_MAX_LENGTH_DIFF , <EOL> min_score = min_score , max_offset = max_offset ) <EOL> with closing ( conn . execute ( PART_SEARCH_SQL , params ) ) as result : <EOL> for row in result : <EOL> matched . append ( row ) <EOL> if row [ '<STR_LIT>' ] >= best_score : <EOL> best_score = row [ '<STR_LIT>' ] <EOL> if best_score > good_enough_score : <EOL> break <EOL> return matched <EOL> class FingerprintSearcher ( object ) : <EOL> def __init__ ( self , db , idx = None , fast = True ) : <EOL> self . db = db <EOL> self . idx = idx <EOL> self . min_score = const . TRACK_GROUP_MERGE_THRESHOLD <EOL> self . max_length_diff = const . FINGERPRINT_MAX_LENGTH_DIFF <EOL> self . max_offset = const . TRACK_MAX_OFFSET <EOL> self . fast = fast <EOL> def _create_search_query ( self , fp , length , condition ) : <EOL> f_columns = [ <EOL> schema . fingerprint . c . id , <EOL> schema . fingerprint . c . track_id , <EOL> sql . func . acoustid_compare2 ( schema . fingerprint . c . fingerprint , fp , <EOL> self . max_offset ) . label ( '<STR_LIT>' ) , <EOL> ] <EOL> f_where = sql . and_ ( <EOL> condition , <EOL> schema . fingerprint . c . length . between ( length - self . max_length_diff , <EOL> length + self . max_length_diff ) ) <EOL> f = sql . select ( f_columns , f_where ) . alias ( '<STR_LIT:f>' ) <EOL> columns = [ f . c . id , f . c . track_id , schema . track . c . gid . label ( '<STR_LIT>' ) , f . c . score ] <EOL> src = f . join ( schema . track , schema . track . c . id == f . c . track_id ) <EOL> return sql . select ( columns , f . c . score > self . min_score , src , <EOL> order_by = [ f . c . score . desc ( ) , f . c . id ] ) <EOL> def _search_index ( self , fp , length ) : <EOL> fp_query = self . db . execute ( sql . select ( [ sql . func . acoustid_extract_query ( fp ) ] ) ) . scalar ( ) <EOL> if not fp_query : <EOL> return [ ] <EOL> with closing ( self . idx . connect ( ) ) as idx : <EOL> results = idx . search ( fp_query ) <EOL> if not results : <EOL> return [ ] <EOL> min_score = results [ <NUM_LIT:0> ] . score * <NUM_LIT:0.1> <EOL> candidate_ids = [ r . id for r in results if r . score > min_score ] <EOL> if not candidate_ids : <EOL> return [ ] <EOL> condition = schema . fingerprint . c . id . in_ ( candidate_ids ) <EOL> query = self . _create_search_query ( fp , length , condition ) <EOL> matches = self . db . execute ( query ) . fetchall ( ) <EOL> return matches <EOL> def _search_database ( self , fp , length , min_fp_id ) : <EOL> condition = sql . func . acoustid_extract_query ( schema . fingerprint . c . fingerprint ) . op ( '<STR_LIT>' ) ( sql . func . acoustid_extract_query ( fp ) ) <EOL> if min_fp_id : <EOL> condition = sql . and_ ( condition , schema . fingerprint . c . id > min_fp_id ) <EOL> query = self . _create_search_query ( fp , length , condition ) <EOL> matches = self . db . execute ( query ) . fetchall ( ) <EOL> return matches <EOL> def _get_min_indexed_fp_id ( self ) : <EOL> if self . idx is None : <EOL> return <NUM_LIT:0> <EOL> with closing ( self . idx . connect ( ) ) as idx : <EOL> return int ( idx . get_attribute ( '<STR_LIT>' ) or '<STR_LIT:0>' ) <EOL> def search ( self , fp , length ) : <EOL> min_fp_id = <NUM_LIT:0> if self . idx is None or self . fast else self . _get_min_indexed_fp_id ( ) <EOL> matches = None <EOL> if self . idx is not None : <EOL> try : <EOL> matches = self . _search_index ( fp , length ) <EOL> except IndexClientError : <EOL> logger . exception ( "<STR_LIT>" ) <EOL> matches = None <EOL> if not self . fast and not matches : <EOL> matches = self . _search_database ( fp , length , min_fp_id ) <EOL> return matches or [ ] <EOL> def insert_fingerprint ( conn , data , submission_id = None , source_id = None ) : <EOL> """<STR_LIT>""" <EOL> with conn . begin ( ) : <EOL> insert_stmt = schema . fingerprint . insert ( ) . values ( { <EOL> '<STR_LIT>' : data [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : data [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : data . get ( '<STR_LIT>' ) , <EOL> '<STR_LIT>' : data . get ( '<STR_LIT>' ) , <EOL> '<STR_LIT>' : data [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : <NUM_LIT:1> , <EOL> } ) <EOL> id = conn . execute ( insert_stmt ) . inserted_primary_key [ <NUM_LIT:0> ] <EOL> if submission_id and source_id : <EOL> insert_stmt = schema . fingerprint_source . insert ( ) . values ( { <EOL> '<STR_LIT>' : id , <EOL> '<STR_LIT>' : submission_id , <EOL> '<STR_LIT>' : source_id , <EOL> } ) <EOL> conn . execute ( insert_stmt ) <EOL> logger . debug ( "<STR_LIT>" , id ) <EOL> return id <EOL> def inc_fingerprint_submission_count ( conn , id , submission_id = None , source_id = None ) : <EOL> update_stmt = schema . fingerprint . update ( ) . where ( schema . fingerprint . c . id == id ) <EOL> conn . execute ( update_stmt . values ( submission_count = sql . text ( '<STR_LIT>' ) ) ) <EOL> if submission_id and source_id : <EOL> insert_stmt = schema . fingerprint_source . insert ( ) . values ( { <EOL> '<STR_LIT>' : id , <EOL> '<STR_LIT>' : submission_id , <EOL> '<STR_LIT>' : source_id , <EOL> } ) <EOL> conn . execute ( insert_stmt ) <EOL> return True <EOL> def update_fingerprint_index ( db , index , limit = <NUM_LIT:1000> ) : <EOL> with closing ( index . connect ( ) ) as index : <EOL> max_id = int ( index . get_attribute ( '<STR_LIT>' ) or '<STR_LIT:0>' ) <EOL> query = sql . select ( [ <EOL> schema . fingerprint . c . id , <EOL> sql . func . acoustid_extract_query ( schema . fingerprint . c . fingerprint ) , <EOL> ] ) . where ( schema . fingerprint . c . id > max_id ) . order_by ( schema . fingerprint . c . id ) . limit ( limit ) <EOL> in_transaction = False <EOL> for id , fingerprint in db . execute ( query ) : <EOL> if not in_transaction : <EOL> index . begin ( ) <EOL> in_transaction = True <EOL> logger . debug ( "<STR_LIT>" , id , index ) <EOL> index . insert ( id , fingerprint ) <EOL> if in_transaction : <EOL> index . commit ( ) </s>
