<s> """<STR_LIT>""" <EOL> from __future__ import print_function <EOL> import time <EOL> import docopt <EOL> import os <EOL> import sys <EOL> import multiprocessing <EOL> import joblib <EOL> import audfprint_analyze <EOL> import hash_table <EOL> import audfprint_match <EOL> def filename_list_iterator ( filelist , wavdir , wavext , listflag ) : <EOL> """<STR_LIT>""" <EOL> if not listflag : <EOL> for filename in filelist : <EOL> yield os . path . join ( wavdir , filename + wavext ) <EOL> else : <EOL> for listfilename in filelist : <EOL> with open ( listfilename , '<STR_LIT:r>' ) as f : <EOL> for filename in f : <EOL> yield os . path . join ( wavdir , filename . rstrip ( '<STR_LIT:\n>' ) + wavext ) <EOL> def ensure_dir ( dirname ) : <EOL> """<STR_LIT>""" <EOL> if len ( dirname ) : <EOL> if not os . path . exists ( dirname ) : <EOL> try : <EOL> os . makedirs ( dirname ) <EOL> except : <EOL> pass <EOL> def file_precompute_peaks_or_hashes ( analyzer , filename , precompdir , <EOL> precompext = None , hashes_not_peaks = True , <EOL> skip_existing = False , <EOL> strip_prefix = None ) : <EOL> """<STR_LIT>""" <EOL> if strip_prefix and filename [ : len ( strip_prefix ) ] == strip_prefix : <EOL> tail_filename = filename [ len ( strip_prefix ) : ] <EOL> else : <EOL> tail_filename = filename <EOL> relname = '<STR_LIT:/>' . join ( [ comp for comp in tail_filename . split ( '<STR_LIT:/>' ) <EOL> if comp != '<STR_LIT:.>' and comp != '<STR_LIT:..>' and comp != '<STR_LIT>' ] ) <EOL> root = os . path . splitext ( relname ) [ <NUM_LIT:0> ] <EOL> if precompext is None : <EOL> if hashes_not_peaks : <EOL> precompext = audfprint_analyze . PRECOMPEXT <EOL> else : <EOL> precompext = audfprint_analyze . PRECOMPPKEXT <EOL> opfname = os . path . join ( precompdir , root + precompext ) <EOL> if skip_existing and os . path . isfile ( opfname ) : <EOL> return [ "<STR_LIT>" + opfname + "<STR_LIT>" ] <EOL> else : <EOL> if hashes_not_peaks : <EOL> type = "<STR_LIT>" <EOL> saver = audfprint_analyze . hashes_save <EOL> output = analyzer . wavfile2hashes ( filename ) <EOL> else : <EOL> type = "<STR_LIT>" <EOL> saver = audfprint_analyze . peaks_save <EOL> output = analyzer . wavfile2peaks ( filename ) <EOL> if len ( output ) == <NUM_LIT:0> : <EOL> message = "<STR_LIT>" + filename + "<STR_LIT>" <EOL> else : <EOL> ensure_dir ( os . path . split ( opfname ) [ <NUM_LIT:0> ] ) <EOL> saver ( opfname , output ) <EOL> message = ( "<STR_LIT>" + opfname + "<STR_LIT>" % ( len ( output ) , type , analyzer . soundfiledur ) ) <EOL> return [ message ] <EOL> def file_precompute ( analyzer , filename , precompdir , type = '<STR_LIT>' , skip_existing = False , strip_prefix = None ) : <EOL> """<STR_LIT>""" <EOL> print ( time . ctime ( ) , "<STR_LIT>" , type , "<STR_LIT>" , filename , "<STR_LIT>" ) <EOL> hashes_not_peaks = ( type == '<STR_LIT>' ) <EOL> return file_precompute_peaks_or_hashes ( analyzer , filename , precompdir , <EOL> hashes_not_peaks = hashes_not_peaks , <EOL> skip_existing = skip_existing , <EOL> strip_prefix = strip_prefix ) <EOL> def make_ht_from_list ( analyzer , filelist , hashbits , depth , maxtime , pipe = None ) : <EOL> """<STR_LIT>""" <EOL> ht = hash_table . HashTable ( hashbits = hashbits , depth = depth , maxtime = maxtime ) <EOL> for filename in filelist : <EOL> hashes = analyzer . wavfile2hashes ( filename ) <EOL> ht . store ( filename , hashes ) <EOL> if pipe : <EOL> pipe . send ( ht ) <EOL> else : <EOL> return ht <EOL> def do_cmd ( cmd , analyzer , hash_tab , filename_iter , matcher , outdir , type , report , skip_existing = False , strip_prefix = None ) : <EOL> """<STR_LIT>""" <EOL> if cmd == '<STR_LIT>' or cmd == '<STR_LIT>' : <EOL> for filename in filename_iter : <EOL> hash_tab2 = hash_table . HashTable ( filename ) <EOL> if "<STR_LIT>" in hash_tab . params : <EOL> assert hash_tab . params [ "<STR_LIT>" ] == hash_tab2 . params [ "<STR_LIT>" ] <EOL> else : <EOL> hash_tab . params [ "<STR_LIT>" ] = hash_tab2 . params [ "<STR_LIT>" ] <EOL> hash_tab . merge ( hash_tab2 ) <EOL> elif cmd == '<STR_LIT>' : <EOL> for filename in filename_iter : <EOL> report ( file_precompute ( analyzer , filename , outdir , type , skip_existing = skip_existing , strip_prefix = strip_prefix ) ) <EOL> elif cmd == '<STR_LIT>' : <EOL> for num , filename in enumerate ( filename_iter ) : <EOL> msgs = matcher . file_match_to_msgs ( analyzer , hash_tab , filename , num ) <EOL> report ( msgs ) <EOL> elif cmd == '<STR_LIT>' or cmd == '<STR_LIT>' : <EOL> tothashes = <NUM_LIT:0> <EOL> ix = <NUM_LIT:0> <EOL> for filename in filename_iter : <EOL> report ( [ time . ctime ( ) + "<STR_LIT>" + str ( ix ) + "<STR_LIT>" <EOL> + filename + "<STR_LIT>" ] ) <EOL> dur , nhash = analyzer . ingest ( hash_tab , filename ) <EOL> tothashes += nhash <EOL> ix += <NUM_LIT:1> <EOL> report ( [ "<STR_LIT>" + str ( tothashes ) + "<STR_LIT>" <EOL> + "<STR_LIT>" % ( tothashes / float ( analyzer . soundfiletotaldur ) ) <EOL> + "<STR_LIT>" ] ) <EOL> elif cmd == '<STR_LIT>' : <EOL> for filename in filename_iter : <EOL> hash_tab . remove ( filename ) <EOL> elif cmd == '<STR_LIT:list>' : <EOL> hash_tab . list ( lambda x : report ( [ x ] ) ) <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" + cmd ) <EOL> def multiproc_add ( analyzer , hash_tab , filename_iter , report , ncores ) : <EOL> """<STR_LIT>""" <EOL> rx = [ [ ] for _ in range ( ncores ) ] <EOL> tx = [ [ ] for _ in range ( ncores ) ] <EOL> pr = [ [ ] for _ in range ( ncores ) ] <EOL> filelists = [ [ ] for _ in range ( ncores ) ] <EOL> ix = <NUM_LIT:0> <EOL> for filename in filename_iter : <EOL> filelists [ ix % ncores ] . append ( filename ) <EOL> ix += <NUM_LIT:1> <EOL> for ix in range ( ncores ) : <EOL> rx [ ix ] , tx [ ix ] = multiprocessing . Pipe ( False ) <EOL> pr [ ix ] = multiprocessing . Process ( target = make_ht_from_list , <EOL> args = ( analyzer , filelists [ ix ] , <EOL> hash_tab . hashbits , <EOL> hash_tab . depth , <EOL> ( <NUM_LIT:1> << hash_tab . maxtimebits ) , <EOL> tx [ ix ] ) ) <EOL> pr [ ix ] . start ( ) <EOL> for core in range ( ncores ) : <EOL> hash_tabx = rx [ core ] . recv ( ) <EOL> report ( [ "<STR_LIT>" + str ( core ) + "<STR_LIT>" <EOL> + str ( len ( hash_tabx . names ) ) <EOL> + "<STR_LIT>" + str ( sum ( hash_tabx . counts ) ) + "<STR_LIT>" ] ) <EOL> hash_tab . merge ( hash_tabx ) <EOL> pr [ core ] . join ( ) <EOL> def matcher_file_match_to_msgs ( matcher , analyzer , hash_tab , filename ) : <EOL> """<STR_LIT>""" <EOL> return matcher . file_match_to_msgs ( analyzer , hash_tab , filename ) <EOL> def do_cmd_multiproc ( cmd , analyzer , hash_tab , filename_iter , matcher , <EOL> outdir , type , report , skip_existing = False , <EOL> strip_prefix = None , ncores = <NUM_LIT:1> ) : <EOL> """<STR_LIT>""" <EOL> if cmd == '<STR_LIT>' : <EOL> msgslist = joblib . Parallel ( n_jobs = ncores ) ( <EOL> joblib . delayed ( file_precompute ) ( analyzer , file , outdir , type , skip_existing , strip_prefix = strip_prefix ) <EOL> for file in filename_iter <EOL> ) <EOL> for msgs in msgslist : <EOL> report ( msgs ) <EOL> elif cmd == '<STR_LIT>' : <EOL> msgslist = joblib . Parallel ( n_jobs = ncores ) ( <EOL> joblib . delayed ( matcher_file_match_to_msgs ) ( matcher , analyzer , <EOL> hash_tab , filename ) <EOL> for filename in filename_iter <EOL> ) <EOL> for msgs in msgslist : <EOL> report ( msgs ) <EOL> elif cmd == '<STR_LIT>' or cmd == '<STR_LIT>' : <EOL> multiproc_add ( analyzer , hash_tab , filename_iter , report , ncores ) <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" + cmd ) <EOL> def setup_analyzer ( args ) : <EOL> """<STR_LIT>""" <EOL> analyzer = audfprint_analyze . Analyzer ( ) <EOL> analyzer . density = float ( args [ '<STR_LIT>' ] ) <EOL> analyzer . maxpksperframe = int ( args [ '<STR_LIT>' ] ) <EOL> analyzer . maxpairsperpeak = int ( args [ '<STR_LIT>' ] ) <EOL> analyzer . f_sd = float ( args [ '<STR_LIT>' ] ) <EOL> analyzer . shifts = int ( args [ '<STR_LIT>' ] ) <EOL> analyzer . target_sr = int ( args [ '<STR_LIT>' ] ) <EOL> analyzer . n_fft = <NUM_LIT> <EOL> analyzer . n_hop = analyzer . n_fft / <NUM_LIT:2> <EOL> if analyzer . shifts == <NUM_LIT:0> : <EOL> analyzer . shifts = <NUM_LIT:4> if args [ '<STR_LIT>' ] else <NUM_LIT:1> <EOL> analyzer . fail_on_error = not args [ '<STR_LIT>' ] <EOL> return analyzer <EOL> def setup_matcher ( args ) : <EOL> """<STR_LIT>""" <EOL> matcher = audfprint_match . Matcher ( ) <EOL> matcher . window = int ( args [ '<STR_LIT>' ] ) <EOL> matcher . threshcount = int ( args [ '<STR_LIT>' ] ) <EOL> matcher . max_returns = int ( args [ '<STR_LIT>' ] ) <EOL> matcher . search_depth = int ( args [ '<STR_LIT>' ] ) <EOL> matcher . sort_by_time = args [ '<STR_LIT>' ] <EOL> matcher . exact_count = args [ '<STR_LIT>' ] | args [ '<STR_LIT>' ] | args [ '<STR_LIT>' ] <EOL> matcher . illustrate = args [ '<STR_LIT>' ] | args [ '<STR_LIT>' ] <EOL> matcher . illustrate_hpf = args [ '<STR_LIT>' ] <EOL> matcher . verbose = args [ '<STR_LIT>' ] <EOL> matcher . find_time_range = args [ '<STR_LIT>' ] <EOL> matcher . time_quantile = float ( args [ '<STR_LIT>' ] ) <EOL> return matcher <EOL> def setup_reporter ( args ) : <EOL> """<STR_LIT>""" <EOL> opfile = args [ '<STR_LIT>' ] <EOL> if opfile and len ( opfile ) : <EOL> f = open ( opfile , "<STR_LIT:w>" ) <EOL> def report ( msglist ) : <EOL> """<STR_LIT>""" <EOL> for msg in msglist : <EOL> f . write ( msg + "<STR_LIT:\n>" ) <EOL> else : <EOL> def report ( msglist ) : <EOL> """<STR_LIT>""" <EOL> for msg in msglist : <EOL> print ( msg ) <EOL> return report <EOL> USAGE = """<STR_LIT>""" <EOL> __version__ = <NUM_LIT> <EOL> def main ( argv ) : <EOL> """<STR_LIT>""" <EOL> args = docopt . docopt ( USAGE , version = __version__ , argv = argv [ <NUM_LIT:1> : ] ) <EOL> poss_cmds = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT:list>' , '<STR_LIT>' ] <EOL> cmdlist = [ cmdname <EOL> for cmdname in poss_cmds <EOL> if args [ cmdname ] ] <EOL> if len ( cmdlist ) != <NUM_LIT:1> : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> cmd = cmdlist [ <NUM_LIT:0> ] <EOL> report = setup_reporter ( args ) <EOL> initticks = time . clock ( ) <EOL> if args [ "<STR_LIT>" ] : <EOL> args [ "<STR_LIT>" ] = int ( args [ "<STR_LIT>" ] ) <EOL> else : <EOL> args [ "<STR_LIT>" ] = hash_table . _bitsfor ( int ( args [ "<STR_LIT>" ] ) ) <EOL> analyzer = setup_analyzer ( args ) if not ( <EOL> cmd is "<STR_LIT>" or cmd is "<STR_LIT>" <EOL> or cmd is "<STR_LIT:list>" or cmd is "<STR_LIT>" ) else None <EOL> precomp_type = '<STR_LIT>' <EOL> if cmd is not "<STR_LIT>" : <EOL> dbasename = args [ '<STR_LIT>' ] <EOL> if not dbasename : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> if cmd == "<STR_LIT>" or cmd == "<STR_LIT>" : <EOL> ensure_dir ( os . path . split ( dbasename ) [ <NUM_LIT:0> ] ) <EOL> hash_tab = hash_table . HashTable ( <EOL> hashbits = int ( args [ '<STR_LIT>' ] ) , <EOL> depth = int ( args [ '<STR_LIT>' ] ) , <EOL> maxtime = ( <NUM_LIT:1> << int ( args [ '<STR_LIT>' ] ) ) ) <EOL> if analyzer : <EOL> hash_tab . params [ '<STR_LIT>' ] = analyzer . target_sr <EOL> else : <EOL> if args [ '<STR_LIT>' ] : <EOL> report ( [ time . ctime ( ) + "<STR_LIT>" + dbasename ] ) <EOL> hash_tab = hash_table . HashTable ( dbasename ) <EOL> if analyzer and '<STR_LIT>' in hash_tab . params and hash_tab . params [ '<STR_LIT>' ] != analyzer . target_sr : <EOL> print ( "<STR_LIT>" , analyzer . target_sr ) <EOL> else : <EOL> hash_tab = None <EOL> if args [ '<STR_LIT>' ] : <EOL> precomp_type = '<STR_LIT>' <EOL> matcher = setup_matcher ( args ) if cmd == '<STR_LIT>' else None <EOL> filename_iter = filename_list_iterator ( <EOL> args [ '<STR_LIT>' ] , args [ '<STR_LIT>' ] , args [ '<STR_LIT>' ] , args [ '<STR_LIT>' ] ) <EOL> ncores = int ( args [ '<STR_LIT>' ] ) <EOL> if ncores > <NUM_LIT:1> and not ( cmd == "<STR_LIT>" or cmd == "<STR_LIT>" or <EOL> cmd == "<STR_LIT:list>" or cmd == "<STR_LIT>" ) : <EOL> do_cmd_multiproc ( cmd , analyzer , hash_tab , filename_iter , <EOL> matcher , args [ '<STR_LIT>' ] , <EOL> precomp_type , report , <EOL> skip_existing = args [ '<STR_LIT>' ] , <EOL> strip_prefix = args [ '<STR_LIT>' ] , <EOL> ncores = ncores ) <EOL> else : <EOL> do_cmd ( cmd , analyzer , hash_tab , filename_iter , <EOL> matcher , args [ '<STR_LIT>' ] , precomp_type , report , <EOL> skip_existing = args [ '<STR_LIT>' ] , <EOL> strip_prefix = args [ '<STR_LIT>' ] ) <EOL> elapsedtime = time . clock ( ) - initticks <EOL> if analyzer and analyzer . soundfiletotaldur > <NUM_LIT:0.> : <EOL> print ( "<STR_LIT>" <EOL> + "<STR_LIT>" % ( analyzer . soundfilecount , analyzer . soundfiletotaldur , <EOL> elapsedtime , ( elapsedtime / analyzer . soundfiletotaldur ) ) ) <EOL> if hash_tab and hash_tab . dirty : <EOL> hash_tab . save ( dbasename ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> main ( sys . argv ) </s>
