<s> import re <EOL> import logging <EOL> import pprint <EOL> import json <EOL> import time <EOL> import operator <EOL> from acoustid import const <EOL> from acoustid . const import MAX_REQUESTS_PER_SECOND <EOL> from acoustid . handler import Handler , Response <EOL> from acoustid . data . track import lookup_mbids , resolve_track_gid , lookup_meta_ids <EOL> from acoustid . data . musicbrainz import lookup_metadata <EOL> from acoustid . data . submission import insert_submission , lookup_submission_status <EOL> from acoustid . data . fingerprint import lookup_fingerprint , decode_fingerprint , FingerprintSearcher <EOL> from acoustid . data . format import find_or_insert_format <EOL> from acoustid . data . application import lookup_application_id_by_apikey <EOL> from acoustid . data . account import lookup_account_id_by_apikey <EOL> from acoustid . data . source import find_or_insert_source <EOL> from acoustid . data . meta import insert_meta , lookup_meta <EOL> from acoustid . data . foreignid import find_or_insert_foreignid <EOL> from acoustid . data . stats import update_lookup_counter , update_user_agent_counter , update_lookup_avg_time <EOL> from acoustid . ratelimiter import RateLimiter <EOL> from werkzeug . exceptions import HTTPException , abort <EOL> from werkzeug . utils import cached_property <EOL> from acoustid . utils import is_uuid , is_foreignid , is_int , check_demo_client_api_key <EOL> from acoustid . api import serialize_response , errors <EOL> logger = logging . getLogger ( __name__ ) <EOL> DEFAULT_FORMAT = '<STR_LIT>' <EOL> FORMATS = set ( [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> DEMO_APPLICATION_ID = <NUM_LIT:2> <EOL> def iter_args_suffixes ( args , * prefixes ) : <EOL> results = set ( ) <EOL> for name in args . iterkeys ( ) : <EOL> for prefix in prefixes : <EOL> if name == prefix : <EOL> results . add ( None ) <EOL> elif name . startswith ( prefix + '<STR_LIT:.>' ) : <EOL> prefix , suffix = name . split ( '<STR_LIT:.>' , <NUM_LIT:1> ) <EOL> if suffix . isdigit ( ) : <EOL> results . add ( int ( suffix ) ) <EOL> return [ '<STR_LIT>' % i if i is not None else '<STR_LIT>' for i in sorted ( results ) ] <EOL> class APIHandlerParams ( object ) : <EOL> def __init__ ( self , config ) : <EOL> self . config = config <EOL> def _parse_client ( self , values , conn ) : <EOL> application_apikey = values . get ( '<STR_LIT>' ) <EOL> if not application_apikey : <EOL> raise errors . MissingParameterError ( '<STR_LIT>' ) <EOL> self . application_id = lookup_application_id_by_apikey ( conn , application_apikey ) <EOL> if not self . application_id : <EOL> if check_demo_client_api_key ( self . config . website . secret , application_apikey ) : <EOL> self . application_id = DEMO_APPLICATION_ID <EOL> else : <EOL> logger . error ( "<STR_LIT>" , application_apikey ) <EOL> raise errors . InvalidAPIKeyError ( ) <EOL> self . application_version = values . get ( '<STR_LIT>' ) <EOL> def _parse_format ( self , values ) : <EOL> self . format = values . get ( '<STR_LIT>' , DEFAULT_FORMAT ) <EOL> if self . format not in FORMATS : <EOL> self . format = DEFAULT_FORMAT <EOL> raise errors . UnknownFormatError ( self . format ) <EOL> if self . format == '<STR_LIT>' : <EOL> callback = values . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> if not re . match ( '<STR_LIT>' , callback ) : <EOL> callback = '<STR_LIT>' <EOL> self . format = '<STR_LIT>' % ( self . format , callback ) <EOL> def parse ( self , values , conn ) : <EOL> self . _parse_format ( values ) <EOL> class APIHandler ( Handler ) : <EOL> params_class = None <EOL> def __init__ ( self , connect = None ) : <EOL> self . _connect = connect <EOL> self . index = None <EOL> self . redis = None <EOL> self . config = None <EOL> self . cluster = None <EOL> @ cached_property <EOL> def conn ( self ) : <EOL> return self . _connect ( ) <EOL> @ classmethod <EOL> def create_from_server ( cls , server ) : <EOL> handler = cls ( connect = server . engine . connect ) <EOL> handler . index = server . index <EOL> handler . redis = server . redis <EOL> handler . config = server . config <EOL> handler . cluster = server . config . cluster <EOL> return handler <EOL> def _error ( self , code , message , format = DEFAULT_FORMAT , status = <NUM_LIT> ) : <EOL> response_data = { <EOL> '<STR_LIT:status>' : '<STR_LIT:error>' , <EOL> '<STR_LIT:error>' : { <EOL> '<STR_LIT:code>' : code , <EOL> '<STR_LIT:message>' : message <EOL> } <EOL> } <EOL> return serialize_response ( response_data , format , status = status ) <EOL> def _ok ( self , data , format = DEFAULT_FORMAT ) : <EOL> response_data = { '<STR_LIT:status>' : '<STR_LIT>' } <EOL> response_data . update ( data ) <EOL> return serialize_response ( response_data , format ) <EOL> def _rate_limit ( self , user_ip , application_id ) : <EOL> ip_rate_limit = self . config . rate_limiter . ips . get ( user_ip , MAX_REQUESTS_PER_SECOND ) <EOL> if self . rate_limiter . limit ( '<STR_LIT>' , user_ip , ip_rate_limit ) : <EOL> if application_id == DEMO_APPLICATION_ID : <EOL> raise errors . TooManyRequests ( ip_rate_limit ) <EOL> if application_id is not None : <EOL> application_rate_limit = self . config . rate_limiter . applications . get ( application_id ) <EOL> if application_rate_limit is not None : <EOL> if self . rate_limiter . limit ( '<STR_LIT>' , application_id , application_rate_limit ) : <EOL> if application_id == DEMO_APPLICATION_ID : <EOL> raise errors . TooManyRequests ( application_rate_limit ) <EOL> def handle ( self , req ) : <EOL> params = self . params_class ( self . config ) <EOL> if req . access_route : <EOL> self . user_ip = req . access_route [ <NUM_LIT:0> ] <EOL> else : <EOL> self . user_ip = req . remote_addr <EOL> self . is_secure = req . is_secure <EOL> self . user_agent = req . user_agent <EOL> self . rate_limiter = RateLimiter ( self . redis , '<STR_LIT>' ) <EOL> try : <EOL> try : <EOL> params . parse ( req . values , self . conn ) <EOL> self . _rate_limit ( self . user_ip , getattr ( params , '<STR_LIT>' , None ) ) <EOL> return self . _ok ( self . _handle_internal ( params ) , params . format ) <EOL> except errors . WebServiceError : <EOL> raise <EOL> except StandardError : <EOL> logger . exception ( '<STR_LIT>' ) <EOL> raise errors . InternalError ( ) <EOL> except errors . WebServiceError , e : <EOL> logger . error ( "<STR_LIT>" , e . message ) <EOL> return self . _error ( e . code , e . message , params . format , status = e . status ) <EOL> class LookupHandlerParams ( APIHandlerParams ) : <EOL> duration_name = '<STR_LIT>' <EOL> def _parse_query ( self , values , suffix ) : <EOL> p = { } <EOL> p [ '<STR_LIT:index>' ] = ( suffix or '<STR_LIT>' ) [ <NUM_LIT:1> : ] <EOL> p [ '<STR_LIT>' ] = values . get ( '<STR_LIT>' + suffix ) <EOL> if p [ '<STR_LIT>' ] and not is_uuid ( p [ '<STR_LIT>' ] ) : <EOL> raise errors . InvalidUUIDError ( '<STR_LIT>' + suffix ) <EOL> if not p [ '<STR_LIT>' ] : <EOL> p [ '<STR_LIT>' ] = values . get ( self . duration_name + suffix , type = int ) <EOL> if not p [ '<STR_LIT>' ] : <EOL> raise errors . MissingParameterError ( self . duration_name + suffix ) <EOL> fingerprint_string = values . get ( '<STR_LIT>' + suffix ) <EOL> if not fingerprint_string : <EOL> raise errors . MissingParameterError ( '<STR_LIT>' + suffix ) <EOL> p [ '<STR_LIT>' ] = decode_fingerprint ( fingerprint_string . encode ( '<STR_LIT:ascii>' , '<STR_LIT:ignore>' ) ) <EOL> if not p [ '<STR_LIT>' ] : <EOL> raise errors . InvalidFingerprintError ( ) <EOL> self . fingerprints . append ( p ) <EOL> def parse ( self , values , conn ) : <EOL> super ( LookupHandlerParams , self ) . parse ( values , conn ) <EOL> self . _parse_client ( values , conn ) <EOL> self . meta = values . get ( '<STR_LIT>' ) <EOL> if self . meta == '<STR_LIT:0>' or not self . meta : <EOL> self . meta = [ ] <EOL> elif self . meta == '<STR_LIT:1>' : <EOL> self . meta = [ '<STR_LIT>' ] <EOL> elif self . meta == '<STR_LIT:2>' : <EOL> self . meta = [ '<STR_LIT>' ] <EOL> else : <EOL> self . meta = self . meta . split ( ) <EOL> self . max_duration_diff = values . get ( '<STR_LIT>' , type = int ) <EOL> if self . max_duration_diff is None : <EOL> self . max_duration_diff = const . FINGERPRINT_MAX_LENGTH_DIFF <EOL> elif self . max_duration_diff > const . FINGERPRINT_MAX_ALLOWED_LENGTH_DIFF or self . max_duration_diff < <NUM_LIT:1> : <EOL> raise errors . InvalidMaxDurationDiffError ( '<STR_LIT>' ) <EOL> self . batch = values . get ( '<STR_LIT>' , type = int ) <EOL> self . fingerprints = [ ] <EOL> suffixes = list ( iter_args_suffixes ( values , '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> if not suffixes : <EOL> raise errors . MissingParameterError ( '<STR_LIT>' ) <EOL> for i , suffix in enumerate ( suffixes ) : <EOL> try : <EOL> self . _parse_query ( values , suffix ) <EOL> except errors . WebServiceError : <EOL> if not self . fingerprints and i + <NUM_LIT:1> == len ( suffixes ) : <EOL> raise <EOL> class LookupHandler ( APIHandler ) : <EOL> params_class = LookupHandlerParams <EOL> recordings_name = '<STR_LIT>' <EOL> def _inject_recording_ids_internal ( self , add = True , add_sources = False ) : <EOL> el_recording = { } <EOL> res_map = { } <EOL> track_mbid_map = lookup_mbids ( self . conn , self . el_result . keys ( ) ) <EOL> for track_id , mbids in track_mbid_map . iteritems ( ) : <EOL> res_map [ track_id ] = [ ] <EOL> for mbid , sources in mbids : <EOL> res_map [ track_id ] . append ( mbid ) <EOL> if add : <EOL> for result_el in self . el_result [ track_id ] : <EOL> recording = { '<STR_LIT:id>' : mbid } <EOL> if add_sources : <EOL> recording [ '<STR_LIT>' ] = sources <EOL> result_el . setdefault ( self . recordings_name , [ ] ) . append ( recording ) <EOL> el_recording . setdefault ( mbid , [ ] ) . append ( recording ) <EOL> else : <EOL> el_recording . setdefault ( mbid , [ ] ) . extend ( self . el_result [ track_id ] ) <EOL> return el_recording , res_map <EOL> def _inject_user_meta_ids_internal ( self , add = True ) : <EOL> el_recording = { } <EOL> track_meta_map = lookup_meta_ids ( self . conn , self . el_result . keys ( ) ) <EOL> for track_id , meta_ids in track_meta_map . iteritems ( ) : <EOL> for meta_id in meta_ids : <EOL> if add : <EOL> for result_el in self . el_result [ track_id ] : <EOL> recording = { } <EOL> result_el . setdefault ( self . recordings_name , [ ] ) . append ( recording ) <EOL> el_recording . setdefault ( meta_id , [ ] ) . append ( recording ) <EOL> else : <EOL> el_recording . setdefault ( meta_id , [ ] ) . extend ( self . el_result [ track_id ] ) <EOL> return el_recording , track_meta_map <EOL> def extract_recording ( self , m , only_id = False ) : <EOL> recording = { '<STR_LIT:id>' : m [ '<STR_LIT>' ] } <EOL> if only_id : <EOL> return recording <EOL> recording [ '<STR_LIT:title>' ] = m [ '<STR_LIT>' ] or '<STR_LIT>' <EOL> if m [ '<STR_LIT>' ] : <EOL> recording [ '<STR_LIT>' ] = m [ '<STR_LIT>' ] <EOL> if m [ '<STR_LIT>' ] : <EOL> recording [ '<STR_LIT>' ] = m [ '<STR_LIT>' ] <EOL> return recording <EOL> def extract_release ( self , m , only_id = False ) : <EOL> release = { '<STR_LIT:id>' : m [ '<STR_LIT>' ] } <EOL> if only_id : <EOL> return release <EOL> release [ '<STR_LIT:title>' ] = m [ '<STR_LIT>' ] or '<STR_LIT>' <EOL> if m [ '<STR_LIT>' ] : <EOL> release [ '<STR_LIT>' ] = m [ '<STR_LIT>' ] <EOL> if m [ '<STR_LIT>' ] : <EOL> release [ '<STR_LIT>' ] = m [ '<STR_LIT>' ] <EOL> if m [ '<STR_LIT>' ] : <EOL> release [ '<STR_LIT>' ] = m [ '<STR_LIT>' ] <EOL> if m [ '<STR_LIT>' ] : <EOL> release [ '<STR_LIT>' ] = [ ] <EOL> for rem in m [ '<STR_LIT>' ] : <EOL> release_event = { } <EOL> if rem [ '<STR_LIT>' ] : <EOL> release_event [ '<STR_LIT>' ] = rem [ '<STR_LIT>' ] <EOL> date = { } <EOL> if rem [ '<STR_LIT>' ] : <EOL> date [ '<STR_LIT>' ] = rem [ '<STR_LIT>' ] <EOL> if rem [ '<STR_LIT>' ] : <EOL> date [ '<STR_LIT>' ] = rem [ '<STR_LIT>' ] <EOL> if rem [ '<STR_LIT>' ] : <EOL> date [ '<STR_LIT>' ] = rem [ '<STR_LIT>' ] <EOL> if date : <EOL> release_event [ '<STR_LIT:date>' ] = date <EOL> release [ '<STR_LIT>' ] . append ( release_event ) <EOL> release . update ( release [ '<STR_LIT>' ] [ <NUM_LIT:0> ] ) <EOL> return release <EOL> def extract_release_group ( self , m , only_id = False ) : <EOL> release_group = { '<STR_LIT:id>' : m [ '<STR_LIT>' ] } <EOL> if only_id : <EOL> return release_group <EOL> release_group [ '<STR_LIT:title>' ] = m [ '<STR_LIT>' ] or '<STR_LIT>' <EOL> if m [ '<STR_LIT>' ] : <EOL> release_group [ '<STR_LIT:type>' ] = m [ '<STR_LIT>' ] <EOL> if m [ '<STR_LIT>' ] : <EOL> release_group [ '<STR_LIT>' ] = m [ '<STR_LIT>' ] <EOL> if m [ '<STR_LIT>' ] : <EOL> release_group [ '<STR_LIT>' ] = m [ '<STR_LIT>' ] <EOL> return release_group <EOL> def _inject_release_groups_internal ( self , meta , parent , metadata ) : <EOL> release_groups = self . _group_release_groups ( metadata , '<STR_LIT>' in meta ) <EOL> parent [ '<STR_LIT>' ] = [ ] <EOL> for release_group , release_group_metadata in release_groups : <EOL> parent [ '<STR_LIT>' ] . append ( release_group ) <EOL> if '<STR_LIT>' in meta or '<STR_LIT>' in meta : <EOL> self . _inject_releases_internal ( meta , release_group , release_group_metadata ) <EOL> if '<STR_LIT>' in meta : <EOL> for release in release_group [ '<STR_LIT>' ] : <EOL> if '<STR_LIT>' in release and release [ '<STR_LIT>' ] == release_group . get ( '<STR_LIT>' ) : <EOL> del release [ '<STR_LIT>' ] <EOL> if '<STR_LIT:title>' in release and release [ '<STR_LIT:title>' ] == release_group . get ( '<STR_LIT:title>' ) : <EOL> del release [ '<STR_LIT:title>' ] <EOL> def _inject_releases_internal ( self , meta , parent , metadata ) : <EOL> releases = self . _group_releases ( metadata , '<STR_LIT>' in meta ) <EOL> parent [ '<STR_LIT>' ] = [ ] <EOL> for release , release_metadata in releases : <EOL> parent [ '<STR_LIT>' ] . append ( release ) <EOL> if '<STR_LIT>' in meta : <EOL> release [ '<STR_LIT>' ] = list ( self . _group_tracks ( release_metadata ) ) <EOL> if '<STR_LIT>' in meta : <EOL> for medium in release [ '<STR_LIT>' ] : <EOL> for track in medium [ '<STR_LIT>' ] : <EOL> if '<STR_LIT>' in track and track [ '<STR_LIT>' ] == release . get ( '<STR_LIT>' ) : <EOL> del track [ '<STR_LIT>' ] <EOL> def inject_recordings ( self , meta ) : <EOL> recording_els = self . _inject_recording_ids_internal ( True , '<STR_LIT>' in meta ) [ <NUM_LIT:0> ] <EOL> load_releases = False <EOL> load_release_groups = False <EOL> if '<STR_LIT>' in meta or '<STR_LIT>' in meta : <EOL> load_releases = True <EOL> if '<STR_LIT>' in meta or '<STR_LIT>' in meta : <EOL> load_releases = True <EOL> load_release_groups = True <EOL> metadata = lookup_metadata ( self . conn , recording_els . keys ( ) , load_releases = load_releases , load_release_groups = load_release_groups ) <EOL> if '<STR_LIT>' in meta and not metadata : <EOL> user_meta_els = self . _inject_user_meta_ids_internal ( True ) [ <NUM_LIT:0> ] <EOL> recording_els . update ( user_meta_els ) <EOL> user_meta = lookup_meta ( self . conn , user_meta_els . keys ( ) ) <EOL> metadata . extend ( user_meta ) <EOL> for recording , recording_metadata in self . _group_recordings ( metadata , '<STR_LIT>' in meta ) : <EOL> if '<STR_LIT>' in meta or '<STR_LIT>' in meta : <EOL> self . _inject_release_groups_internal ( meta , recording , recording_metadata ) <EOL> if '<STR_LIT>' in meta : <EOL> for release_group in recording . get ( '<STR_LIT>' , [ ] ) : <EOL> for release in release_group . get ( '<STR_LIT>' , [ ] ) : <EOL> for medium in release . get ( '<STR_LIT>' , [ ] ) : <EOL> for track in medium [ '<STR_LIT>' ] : <EOL> if '<STR_LIT:title>' in track and track [ '<STR_LIT:title>' ] == recording . get ( '<STR_LIT:title>' ) : <EOL> del track [ '<STR_LIT:title>' ] <EOL> if '<STR_LIT>' in release_group and release_group [ '<STR_LIT>' ] == recording . get ( '<STR_LIT>' ) : <EOL> del release_group [ '<STR_LIT>' ] <EOL> elif '<STR_LIT>' in meta or '<STR_LIT>' in meta : <EOL> self . _inject_releases_internal ( meta , recording , recording_metadata ) <EOL> if '<STR_LIT>' in meta : <EOL> for release in recording . get ( '<STR_LIT>' , [ ] ) : <EOL> for medium in release . get ( '<STR_LIT>' , [ ] ) : <EOL> for track in medium [ '<STR_LIT>' ] : <EOL> if '<STR_LIT:title>' in track and track [ '<STR_LIT:title>' ] == recording . get ( '<STR_LIT:title>' ) : <EOL> del track [ '<STR_LIT:title>' ] <EOL> if '<STR_LIT>' in release and release [ '<STR_LIT>' ] == recording . get ( '<STR_LIT>' ) : <EOL> del release [ '<STR_LIT>' ] <EOL> for recording_el in recording_els [ recording [ '<STR_LIT:id>' ] ] : <EOL> if '<STR_LIT>' in meta : <EOL> if isinstance ( recording [ '<STR_LIT:id>' ] , int ) : <EOL> del recording [ '<STR_LIT:id>' ] <EOL> if '<STR_LIT:title>' in recording and not recording [ '<STR_LIT:title>' ] : <EOL> del recording [ '<STR_LIT:title>' ] <EOL> if '<STR_LIT>' in recording : <EOL> releasegroups = [ ] <EOL> for releasegroup in recording [ '<STR_LIT>' ] : <EOL> del releasegroup [ '<STR_LIT:id>' ] <EOL> if '<STR_LIT:title>' in releasegroup and not releasegroup [ '<STR_LIT:title>' ] : <EOL> del releasegroup [ '<STR_LIT:title>' ] <EOL> if '<STR_LIT>' in releasegroup : <EOL> releases = [ ] <EOL> for release in releasegroup [ '<STR_LIT>' ] : <EOL> del release [ '<STR_LIT:id>' ] <EOL> if '<STR_LIT:title>' in release and not release [ '<STR_LIT:title>' ] : <EOL> del release [ '<STR_LIT:title>' ] <EOL> if release : <EOL> releases . append ( release ) <EOL> if releases : <EOL> releasegroup [ '<STR_LIT>' ] = releases <EOL> else : <EOL> del releasegroup [ '<STR_LIT>' ] <EOL> if releasegroup : <EOL> releasegroups . append ( releasegroup ) <EOL> if releasegroups : <EOL> recording [ '<STR_LIT>' ] = releasegroups <EOL> else : <EOL> del recording [ '<STR_LIT>' ] <EOL> if '<STR_LIT>' in recording : <EOL> releases = [ ] <EOL> for release in recording [ '<STR_LIT>' ] : <EOL> del release [ '<STR_LIT:id>' ] <EOL> if '<STR_LIT:title>' in release and not release [ '<STR_LIT:title>' ] : <EOL> del release [ '<STR_LIT:title>' ] <EOL> if release : <EOL> releases . append ( release ) <EOL> if releases : <EOL> recording [ '<STR_LIT>' ] = releases <EOL> else : <EOL> del recording [ '<STR_LIT>' ] <EOL> recording_el . update ( recording ) <EOL> def inject_releases ( self , meta ) : <EOL> recording_els , track_mbid_map = self . _inject_recording_ids_internal ( False ) <EOL> metadata = lookup_metadata ( self . conn , recording_els . keys ( ) , load_releases = True , load_release_groups = True ) <EOL> for track_id , track_metadata in self . _group_metadata ( metadata , track_mbid_map ) : <EOL> result = { } <EOL> self . _inject_releases_internal ( meta , result , track_metadata ) <EOL> for result_el in self . el_result [ track_id ] : <EOL> result_el . update ( result ) <EOL> def inject_release_groups ( self , meta ) : <EOL> recording_els , track_mbid_map = self . _inject_recording_ids_internal ( False ) <EOL> metadata = lookup_metadata ( self . conn , recording_els . keys ( ) , load_releases = True , load_release_groups = True ) <EOL> for track_id , track_metadata in self . _group_metadata ( metadata , track_mbid_map ) : <EOL> result = { } <EOL> self . _inject_release_groups_internal ( meta , result , track_metadata ) <EOL> for result_el in self . el_result [ track_id ] : <EOL> result_el . update ( result ) <EOL> def _group_metadata ( self , metadata , track_mbid_map ) : <EOL> results = { } <EOL> for track_id , mbids in track_mbid_map . iteritems ( ) : <EOL> mbids = set ( mbids ) <EOL> results [ track_id ] = [ ] <EOL> for item in metadata : <EOL> if item [ '<STR_LIT>' ] in mbids : <EOL> results [ track_id ] . append ( item ) <EOL> return results . iteritems ( ) <EOL> def _group_release_groups ( self , metadata , only_ids = False ) : <EOL> results = { } <EOL> for item in metadata : <EOL> id = item [ '<STR_LIT>' ] <EOL> if id not in results : <EOL> results [ id ] = ( self . extract_release_group ( item , only_id = only_ids ) , [ ] ) <EOL> results [ id ] [ <NUM_LIT:1> ] . append ( item ) <EOL> return results . itervalues ( ) <EOL> def _group_recordings ( self , metadata , only_ids = False ) : <EOL> results = { } <EOL> for item in metadata : <EOL> id = item [ '<STR_LIT>' ] <EOL> if id not in results : <EOL> results [ id ] = ( self . extract_recording ( item , only_id = only_ids ) , [ ] ) <EOL> results [ id ] [ <NUM_LIT:1> ] . append ( item ) <EOL> return results . itervalues ( ) <EOL> def _group_releases ( self , metadata , only_ids = False ) : <EOL> results = { } <EOL> for item in metadata : <EOL> id = item [ '<STR_LIT>' ] <EOL> if id not in results : <EOL> results [ id ] = ( self . extract_release ( item , only_id = only_ids ) , [ ] ) <EOL> results [ id ] [ <NUM_LIT:1> ] . append ( item ) <EOL> return results . itervalues ( ) <EOL> def _group_tracks ( self , metadata ) : <EOL> results = { } <EOL> for item in metadata : <EOL> medium_pos = item [ '<STR_LIT>' ] <EOL> medium = results . get ( medium_pos ) <EOL> if medium is None : <EOL> medium = { '<STR_LIT>' : medium_pos , '<STR_LIT>' : [ ] } <EOL> medium [ '<STR_LIT>' ] = item [ '<STR_LIT>' ] <EOL> if item [ '<STR_LIT>' ] : <EOL> medium [ '<STR_LIT>' ] = item [ '<STR_LIT>' ] <EOL> if item [ '<STR_LIT>' ] : <EOL> medium [ '<STR_LIT:title>' ] = item [ '<STR_LIT>' ] <EOL> results [ medium_pos ] = medium <EOL> track = { <EOL> '<STR_LIT:id>' : item [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : item [ '<STR_LIT>' ] , <EOL> '<STR_LIT:title>' : item [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : item [ '<STR_LIT>' ] , <EOL> } <EOL> medium [ '<STR_LIT>' ] . append ( track ) <EOL> return results . itervalues ( ) <EOL> def inject_m2 ( self , meta ) : <EOL> el_recording = self . _inject_recording_ids_internal ( True ) [ <NUM_LIT:0> ] <EOL> metadata = lookup_metadata ( self . conn , el_recording . keys ( ) , load_releases = True ) <EOL> last_recording_id = None <EOL> for item in metadata : <EOL> if last_recording_id != item [ '<STR_LIT>' ] : <EOL> recording = self . extract_recording ( item , True ) <EOL> last_recording_id = recording [ '<STR_LIT:id>' ] <EOL> for el in el_recording [ recording [ '<STR_LIT:id>' ] ] : <EOL> if item [ '<STR_LIT>' ] : <EOL> recording [ '<STR_LIT>' ] = item [ '<STR_LIT>' ] <EOL> recording [ '<STR_LIT>' ] = [ ] <EOL> el . update ( recording ) <EOL> metadata = sorted ( metadata , key = operator . itemgetter ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> for item in metadata : <EOL> for el in el_recording [ item [ '<STR_LIT>' ] ] : <EOL> medium = { <EOL> '<STR_LIT>' : item [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : item [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT:id>' : item [ '<STR_LIT>' ] , <EOL> '<STR_LIT:title>' : item [ '<STR_LIT>' ] , <EOL> } , <EOL> } <EOL> if item [ '<STR_LIT>' ] : <EOL> medium [ '<STR_LIT>' ] = item [ '<STR_LIT>' ] <EOL> el [ '<STR_LIT>' ] . append ( { <EOL> '<STR_LIT:title>' : item [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : item [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : item [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : item [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : medium , <EOL> } ) <EOL> def inject_metadata ( self , meta , result_map ) : <EOL> self . el_result = result_map <EOL> if '<STR_LIT>' in meta : <EOL> self . inject_m2 ( meta ) <EOL> elif '<STR_LIT>' in meta or '<STR_LIT>' in meta : <EOL> self . inject_recordings ( meta ) <EOL> elif '<STR_LIT>' in meta or '<STR_LIT>' in meta : <EOL> self . inject_release_groups ( meta ) <EOL> elif '<STR_LIT>' in meta or '<STR_LIT>' in meta : <EOL> self . inject_releases ( meta ) <EOL> def _inject_results ( self , results , result_map , matches ) : <EOL> seen = set ( ) <EOL> for fingerprint_id , track_id , track_gid , score in matches : <EOL> if track_id in seen : <EOL> continue <EOL> seen . add ( track_id ) <EOL> result = { '<STR_LIT:id>' : track_gid , '<STR_LIT>' : score } <EOL> result_map . setdefault ( track_id , [ ] ) . append ( result ) <EOL> results . append ( result ) <EOL> return seen <EOL> def _handle_internal ( self , params ) : <EOL> import time <EOL> t = time . time ( ) <EOL> update_user_agent_counter ( self . redis , params . application_id , self . user_agent , self . user_ip ) <EOL> searcher = FingerprintSearcher ( self . conn , self . index ) <EOL> searcher . max_length_diff = params . max_duration_diff <EOL> if params . batch : <EOL> fingerprints = params . fingerprints <EOL> else : <EOL> fingerprints = params . fingerprints [ : <NUM_LIT:1> ] <EOL> all_matches = [ ] <EOL> for p in fingerprints : <EOL> if p [ '<STR_LIT>' ] : <EOL> track_id = resolve_track_gid ( self . conn , p [ '<STR_LIT>' ] ) <EOL> matches = [ ( <NUM_LIT:0> , track_id , p [ '<STR_LIT>' ] , <NUM_LIT:1.0> ) ] <EOL> else : <EOL> matches = searcher . search ( p [ '<STR_LIT>' ] , p [ '<STR_LIT>' ] ) <EOL> all_matches . append ( matches ) <EOL> response = { } <EOL> if params . batch : <EOL> response [ '<STR_LIT>' ] = fps = [ ] <EOL> result_map = { } <EOL> for p , matches in zip ( fingerprints , all_matches ) : <EOL> results = [ ] <EOL> fps . append ( { '<STR_LIT:index>' : p [ '<STR_LIT:index>' ] , '<STR_LIT>' : results } ) <EOL> track_ids = self . _inject_results ( results , result_map , matches ) <EOL> update_lookup_counter ( self . redis , params . application_id , bool ( track_ids ) ) <EOL> logger . debug ( "<STR_LIT>" , params . application_id , list ( track_ids ) ) <EOL> else : <EOL> response [ '<STR_LIT>' ] = results = [ ] <EOL> result_map = { } <EOL> self . _inject_results ( results , result_map , all_matches [ <NUM_LIT:0> ] ) <EOL> update_lookup_counter ( self . redis , params . application_id , bool ( result_map ) ) <EOL> logger . debug ( "<STR_LIT>" , params . application_id , result_map . keys ( ) ) <EOL> if params . meta and result_map : <EOL> self . inject_metadata ( params . meta , result_map ) <EOL> if fingerprints : <EOL> time_per_fp = ( time . time ( ) - t ) / len ( fingerprints ) <EOL> update_lookup_avg_time ( self . redis , time_per_fp ) <EOL> return response <EOL> class SubmissionStatusHandlerParams ( APIHandlerParams ) : <EOL> def parse ( self , values , conn ) : <EOL> super ( SubmissionStatusHandlerParams , self ) . parse ( values , conn ) <EOL> self . _parse_client ( values , conn ) <EOL> self . ids = values . getlist ( '<STR_LIT:id>' , type = int ) <EOL> class SubmissionStatusHandler ( APIHandler ) : <EOL> params_class = SubmissionStatusHandlerParams <EOL> def _handle_internal ( self , params ) : <EOL> response = { '<STR_LIT>' : [ { '<STR_LIT:id>' : id , '<STR_LIT:status>' : '<STR_LIT>' } for id in params . ids ] } <EOL> tracks = lookup_submission_status ( self . conn , params . ids ) <EOL> for submission in response [ '<STR_LIT>' ] : <EOL> id = submission [ '<STR_LIT:id>' ] <EOL> track_gid = tracks . get ( id ) <EOL> if track_gid is not None : <EOL> submission [ '<STR_LIT:status>' ] = '<STR_LIT>' <EOL> submission [ '<STR_LIT:result>' ] = { '<STR_LIT:id>' : track_gid } <EOL> return response <EOL> class SubmitHandlerParams ( APIHandlerParams ) : <EOL> def _parse_user ( self , values , conn ) : <EOL> account_apikey = values . get ( '<STR_LIT:user>' ) <EOL> if not account_apikey : <EOL> raise errors . MissingParameterError ( '<STR_LIT:user>' ) <EOL> self . account_id = lookup_account_id_by_apikey ( conn , account_apikey ) <EOL> if not self . account_id : <EOL> raise errors . InvalidUserAPIKeyError ( ) <EOL> def _parse_duration_and_format ( self , p , values , suffix ) : <EOL> p [ '<STR_LIT>' ] = values . get ( '<STR_LIT>' + suffix , type = int ) <EOL> if p [ '<STR_LIT>' ] is None : <EOL> raise errors . MissingParameterError ( '<STR_LIT>' + suffix ) <EOL> if p [ '<STR_LIT>' ] <= <NUM_LIT:0> or p [ '<STR_LIT>' ] > <NUM_LIT> : <EOL> raise errors . InvalidDurationError ( '<STR_LIT>' + suffix ) <EOL> p [ '<STR_LIT>' ] = values . get ( '<STR_LIT>' + suffix ) <EOL> def _parse_submission ( self , values , suffix ) : <EOL> p = { } <EOL> p [ '<STR_LIT:index>' ] = ( suffix or '<STR_LIT>' ) [ <NUM_LIT:1> : ] <EOL> p [ '<STR_LIT>' ] = values . get ( '<STR_LIT>' + suffix ) <EOL> if p [ '<STR_LIT>' ] and not is_uuid ( p [ '<STR_LIT>' ] ) : <EOL> raise errors . InvalidUUIDError ( '<STR_LIT>' + suffix ) <EOL> p [ '<STR_LIT>' ] = values . get ( '<STR_LIT>' + suffix ) <EOL> if p [ '<STR_LIT>' ] and not is_foreignid ( p [ '<STR_LIT>' ] ) : <EOL> raise errors . InvalidForeignIDError ( '<STR_LIT>' + suffix ) <EOL> p [ '<STR_LIT>' ] = values . getlist ( '<STR_LIT>' + suffix ) <EOL> if p [ '<STR_LIT>' ] and not all ( map ( is_uuid , p [ '<STR_LIT>' ] ) ) : <EOL> raise errors . InvalidUUIDError ( '<STR_LIT>' + suffix ) <EOL> self . _parse_duration_and_format ( p , values , suffix ) <EOL> fingerprint_string = values . get ( '<STR_LIT>' + suffix ) <EOL> if not fingerprint_string : <EOL> raise errors . MissingParameterError ( '<STR_LIT>' + suffix ) <EOL> p [ '<STR_LIT>' ] = decode_fingerprint ( fingerprint_string . encode ( '<STR_LIT:ascii>' , '<STR_LIT:ignore>' ) ) <EOL> if not p [ '<STR_LIT>' ] : <EOL> raise errors . InvalidFingerprintError ( ) <EOL> p [ '<STR_LIT>' ] = values . get ( '<STR_LIT>' + suffix , type = int ) or None <EOL> if p [ '<STR_LIT>' ] is not None and p [ '<STR_LIT>' ] <= <NUM_LIT:0> : <EOL> raise errors . InvalidBitrateError ( '<STR_LIT>' + suffix ) <EOL> p [ '<STR_LIT>' ] = values . get ( '<STR_LIT>' + suffix ) <EOL> p [ '<STR_LIT>' ] = values . get ( '<STR_LIT>' + suffix ) <EOL> p [ '<STR_LIT>' ] = values . get ( '<STR_LIT>' + suffix ) <EOL> p [ '<STR_LIT>' ] = values . get ( '<STR_LIT>' + suffix ) <EOL> p [ '<STR_LIT>' ] = values . get ( '<STR_LIT>' + suffix , type = int ) <EOL> p [ '<STR_LIT>' ] = values . get ( '<STR_LIT>' + suffix , type = int ) <EOL> p [ '<STR_LIT>' ] = values . get ( '<STR_LIT>' + suffix , type = int ) <EOL> self . submissions . append ( p ) <EOL> def parse ( self , values , conn ) : <EOL> super ( SubmitHandlerParams , self ) . parse ( values , conn ) <EOL> self . _parse_client ( values , conn ) <EOL> self . _parse_user ( values , conn ) <EOL> self . wait = values . get ( '<STR_LIT>' , type = int , default = <NUM_LIT:0> ) <EOL> self . submissions = [ ] <EOL> suffixes = list ( iter_args_suffixes ( values , '<STR_LIT>' ) ) <EOL> if not suffixes : <EOL> raise errors . MissingParameterError ( '<STR_LIT>' ) <EOL> for i , suffix in enumerate ( suffixes ) : <EOL> try : <EOL> self . _parse_submission ( values , suffix ) <EOL> except errors . WebServiceError : <EOL> if not self . submissions and i + <NUM_LIT:1> == len ( suffixes ) : <EOL> raise <EOL> class SubmitHandler ( APIHandler ) : <EOL> params_class = SubmitHandlerParams <EOL> meta_fields = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' ) <EOL> def _handle_internal ( self , params ) : <EOL> response = { '<STR_LIT>' : [ ] } <EOL> ids = set ( ) <EOL> with self . conn . begin ( ) : <EOL> source_id = find_or_insert_source ( self . conn , params . application_id , params . account_id , params . application_version ) <EOL> format_ids = { } <EOL> for p in params . submissions : <EOL> if p [ '<STR_LIT>' ] : <EOL> if p [ '<STR_LIT>' ] not in format_ids : <EOL> format_ids [ p [ '<STR_LIT>' ] ] = find_or_insert_format ( self . conn , p [ '<STR_LIT>' ] ) <EOL> p [ '<STR_LIT>' ] = format_ids [ p [ '<STR_LIT>' ] ] <EOL> for p in params . submissions : <EOL> mbids = p [ '<STR_LIT>' ] or [ None ] <EOL> for mbid in mbids : <EOL> values = { <EOL> '<STR_LIT>' : mbid or None , <EOL> '<STR_LIT>' : p [ '<STR_LIT>' ] or None , <EOL> '<STR_LIT>' : p [ '<STR_LIT>' ] or None , <EOL> '<STR_LIT>' : p [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : p [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : p . get ( '<STR_LIT>' ) , <EOL> '<STR_LIT>' : source_id <EOL> } <EOL> meta_values = dict ( ( n , p [ n ] or None ) for n in self . meta_fields ) <EOL> if any ( meta_values . itervalues ( ) ) : <EOL> values [ '<STR_LIT>' ] = insert_meta ( self . conn , meta_values ) <EOL> if p [ '<STR_LIT>' ] : <EOL> values [ '<STR_LIT>' ] = find_or_insert_foreignid ( self . conn , p [ '<STR_LIT>' ] ) <EOL> id = insert_submission ( self . conn , values ) <EOL> ids . add ( id ) <EOL> submission = { '<STR_LIT:id>' : id , '<STR_LIT:status>' : '<STR_LIT>' } <EOL> if p [ '<STR_LIT:index>' ] : <EOL> submission [ '<STR_LIT:index>' ] = p [ '<STR_LIT:index>' ] <EOL> response [ '<STR_LIT>' ] . append ( submission ) <EOL> if self . redis is not None : <EOL> self . redis . publish ( '<STR_LIT>' , json . dumps ( list ( ids ) ) ) <EOL> clients_waiting_key = '<STR_LIT>' <EOL> clients_waiting = self . redis . incr ( clients_waiting_key ) - <NUM_LIT:1> <EOL> try : <EOL> max_wait = <NUM_LIT:10> <EOL> self . redis . expire ( clients_waiting_key , max_wait ) <EOL> tracks = { } <EOL> remaining = min ( max ( <NUM_LIT:0> , max_wait - <NUM_LIT:2> ** clients_waiting ) , params . wait ) <EOL> logger . debug ( '<STR_LIT>' , remaining , clients_waiting ) <EOL> while remaining > <NUM_LIT:0> and ids : <EOL> logger . debug ( '<STR_LIT>' , remaining ) <EOL> time . sleep ( <NUM_LIT:0.5> ) <EOL> remaining -= <NUM_LIT:0.5> <EOL> tracks = lookup_submission_status ( self . conn , ids ) <EOL> if not tracks : <EOL> continue <EOL> for submission in response [ '<STR_LIT>' ] : <EOL> id = submission [ '<STR_LIT:id>' ] <EOL> track_gid = tracks . get ( id ) <EOL> if track_gid is not None : <EOL> submission [ '<STR_LIT:status>' ] = '<STR_LIT>' <EOL> submission [ '<STR_LIT:result>' ] = { '<STR_LIT:id>' : track_gid } <EOL> ids . remove ( id ) <EOL> finally : <EOL> self . redis . decr ( clients_waiting_key ) <EOL> return response </s>
