<s> import os <EOL> import re <EOL> import time <EOL> import numpy as np <EOL> from os . path import join as pjoin <EOL> import dataLoader as dl <EOL> import cPickle as pickle <EOL> from joblib import Parallel , delayed <EOL> from decoder . decoder_config import SPACE , SCAIL_DATA_DIR , INPUT_DIM , RAW_DIM , DATASET , DATA_SUBSET , SPECIALS_LIST <EOL> from cluster . config import NUM_CPUS , CLUSTER_DIR , PYTHON_CMD , SLEEP_SEC <EOL> from cluster . utils import get_free_nodes , run_cpu_job <EOL> from decoder . decoder_utils import decode <EOL> if DATA_SUBSET == '<STR_LIT>' : <EOL> from decoder . decoder_config import SWBD_SUBSET <EOL> from model_utils import get_model_class_and_params <EOL> from run_utils import load_config <EOL> from optimizer import OptimizerHyperparams <EOL> from errorAnalysis import replace_contractions <EOL> '''<STR_LIT>''' <EOL> def get_char_map ( dataDir ) : <EOL> kaldi_base = '<STR_LIT:/>' . join ( dataDir . split ( '<STR_LIT:/>' ) [ : - <NUM_LIT:3> ] ) + '<STR_LIT:/>' <EOL> with open ( kaldi_base + '<STR_LIT>' , '<STR_LIT:r>' ) as fid : <EOL> labels = [ l . strip ( ) . split ( ) for l in fid . readlines ( ) ] <EOL> labels = dict ( ( int ( k ) , v ) for v , k in labels ) <EOL> return labels <EOL> PATTERN = re . compile ( '<STR_LIT>' , re . UNICODE ) <EOL> MODEL_TYPE = '<STR_LIT>' <EOL> LIKELIHOODS_DIR = pjoin ( SCAIL_DATA_DIR , '<STR_LIT>' % ( DATASET , DATA_SUBSET ) ) <EOL> def decode_utterance ( k , probs , labels , phone_map , lm = None ) : <EOL> labels = np . array ( labels , dtype = np . int32 ) <EOL> probs = probs . astype ( np . float64 ) <EOL> alpha = <NUM_LIT> <EOL> beta = <NUM_LIT> <EOL> hyp0 , hypscore , truescore , align = decode ( probs , <EOL> alpha = alpha , beta = beta , beam = <NUM_LIT> , method = '<STR_LIT>' , clm = lm ) <EOL> ref = [ ] <EOL> for a in labels : <EOL> token = phone_map [ a ] <EOL> if ( token == SPACE and len ( ref ) > <NUM_LIT:0> and ref [ - <NUM_LIT:1> ] != SPACE ) or ( token not in SPECIALS_LIST and PATTERN . match ( token ) ) : <EOL> ref . append ( token ) <EOL> while len ( ref ) and ref [ - <NUM_LIT:1> ] == SPACE : <EOL> ref = ref [ : - <NUM_LIT:1> ] <EOL> while len ( ref ) and ref [ <NUM_LIT:0> ] == SPACE : <EOL> ref = ref [ <NUM_LIT:1> : ] <EOL> hyp = [ ] <EOL> for a in hyp0 : <EOL> token = phone_map [ a ] <EOL> if ( token == SPACE and len ( hyp ) > <NUM_LIT:0> and hyp [ - <NUM_LIT:1> ] != SPACE ) or ( token not in SPECIALS_LIST and PATTERN . match ( token ) ) : <EOL> hyp . append ( token ) <EOL> while len ( hyp ) and hyp [ - <NUM_LIT:1> ] == SPACE : <EOL> hyp = hyp [ : - <NUM_LIT:1> ] <EOL> while len ( hyp ) and hyp [ <NUM_LIT:0> ] == SPACE : <EOL> hyp = hyp [ <NUM_LIT:1> : ] <EOL> return ( hyp , ref , hypscore , truescore , align ) <EOL> def runSeq ( opts ) : <EOL> fid = open ( opts . out_file , '<STR_LIT:w>' ) <EOL> phone_map = get_char_map ( opts . dataDir ) <EOL> print phone_map <EOL> print len ( phone_map ) <EOL> alisDir = opts . alisDir if opts . alisDir else opts . dataDir <EOL> loader = dl . DataLoader ( opts . dataDir , opts . rawDim , opts . inputDim , alisDir ) <EOL> hyps = list ( ) <EOL> refs = list ( ) <EOL> hypscores = list ( ) <EOL> refscores = list ( ) <EOL> numphones = list ( ) <EOL> subsets = list ( ) <EOL> alignments = list ( ) <EOL> if MODEL_TYPE != '<STR_LIT>' : <EOL> cfg_file = '<STR_LIT>' <EOL> params_file = '<STR_LIT>' <EOL> cfg = load_config ( cfg_file ) <EOL> model_class , model_hps = get_model_class_and_params ( MODEL_TYPE ) <EOL> opt_hps = OptimizerHyperparams ( ) <EOL> model_hps . set_from_dict ( cfg ) <EOL> opt_hps . set_from_dict ( cfg ) <EOL> clm = model_class ( None , model_hps , opt_hps , train = False , opt = '<STR_LIT>' ) <EOL> with open ( params_file , '<STR_LIT:rb>' ) as fin : <EOL> clm . from_file ( fin ) <EOL> else : <EOL> from srilm import LM <EOL> from decoder_config import LM_ARPA_FILE <EOL> print '<STR_LIT>' % LM_ARPA_FILE <EOL> clm = LM ( LM_ARPA_FILE ) <EOL> print '<STR_LIT>' <EOL> for i in range ( opts . start_file , opts . start_file + opts . numFiles ) : <EOL> data_dict , alis , keys , _ = loader . loadDataFileDict ( i ) <EOL> keys = sorted ( keys ) <EOL> if DATA_SUBSET == '<STR_LIT>' : <EOL> if SWBD_SUBSET == '<STR_LIT>' : <EOL> keys = [ k for k in keys if k . startswith ( '<STR_LIT>' ) ] <EOL> elif SWBD_SUBSET == '<STR_LIT>' : <EOL> keys = [ k for k in keys if k . startswith ( '<STR_LIT>' ) ] <EOL> ll_file = pjoin ( LIKELIHOODS_DIR , '<STR_LIT>' % i ) <EOL> ll_fid = open ( ll_file , '<STR_LIT:rb>' ) <EOL> probs_dict = pickle . load ( ll_fid ) <EOL> print '<STR_LIT>' % ( NUM_CPUS , i ) <EOL> decoded_utts = Parallel ( n_jobs = NUM_CPUS ) ( delayed ( decode_utterance ) ( k , probs_dict [ k ] , alis [ k ] , phone_map , lm = clm ) for k in keys ) <EOL> for k , ( hyp , ref , hypscore , refscore , align ) in zip ( keys , decoded_utts ) : <EOL> if refscore is None : <EOL> refscore = <NUM_LIT:0.0> <EOL> if hypscore is None : <EOL> hypscore = <NUM_LIT:0.0> <EOL> hyp = replace_contractions ( hyp ) <EOL> fid . write ( k + '<STR_LIT:U+0020>' + '<STR_LIT:U+0020>' . join ( hyp ) + '<STR_LIT:\n>' ) <EOL> hyps . append ( hyp ) <EOL> refs . append ( ref ) <EOL> hypscores . append ( hypscore ) <EOL> refscores . append ( refscore ) <EOL> numphones . append ( len ( alis [ k ] ) ) <EOL> subsets . append ( '<STR_LIT>' if k . startswith ( '<STR_LIT>' ) else '<STR_LIT>' ) <EOL> alignments . append ( align ) <EOL> fid . close ( ) <EOL> pkid = open ( opts . out_file . replace ( '<STR_LIT>' , '<STR_LIT>' ) , '<STR_LIT:wb>' ) <EOL> pickle . dump ( hyps , pkid ) <EOL> pickle . dump ( refs , pkid ) <EOL> pickle . dump ( hypscores , pkid ) <EOL> pickle . dump ( refscores , pkid ) <EOL> pickle . dump ( numphones , pkid ) <EOL> pickle . dump ( subsets , pkid ) <EOL> pickle . dump ( alignments , pkid ) <EOL> pkid . close ( ) <EOL> def runNode ( node , job , opts ) : <EOL> alisDir = opts . alisDir if opts . alisDir else opts . dataDir <EOL> cmd = '<STR_LIT>' % ( PYTHON_CMD , opts . dataDir , alisDir , job , opts . out_file , job ) <EOL> print cmd <EOL> full_cmd = '<STR_LIT>' % ( CLUSTER_DIR , DATASET ) <EOL> full_cmd += '<STR_LIT>' + cmd <EOL> print full_cmd <EOL> log_file = '<STR_LIT>' % ( DATASET , job ) <EOL> run_cpu_job ( node , full_cmd , stdout = open ( log_file , '<STR_LIT:w>' ) , blocking = False ) <EOL> return None <EOL> def runParallel ( opts ) : <EOL> unsub_jobs = list ( ) <EOL> for i in xrange ( opts . start_file , opts . start_file + opts . numFiles ) : <EOL> unsub_jobs . append ( i ) <EOL> unsub_jobs = sorted ( unsub_jobs , key = lambda x : os . path . getsize ( pjoin ( LIKELIHOODS_DIR , '<STR_LIT>' % x ) ) ) <EOL> while len ( unsub_jobs ) : <EOL> free_nodes = get_free_nodes ( '<STR_LIT>' ) <EOL> for free_node in free_nodes : <EOL> if len ( unsub_jobs ) : <EOL> job = unsub_jobs . pop ( ) <EOL> runNode ( free_node , job , opts ) <EOL> if len ( unsub_jobs ) : <EOL> print '<STR_LIT:->' * <NUM_LIT> <EOL> print '<STR_LIT>' % len ( unsub_jobs ) <EOL> print '<STR_LIT:->' * <NUM_LIT> <EOL> time . sleep ( SLEEP_SEC ) <EOL> def jobs_left ( ) : <EOL> wait_count = <NUM_LIT:0> <EOL> for i in xrange ( opts . start_file , opts . start_file + opts . numFiles ) : <EOL> fi = '<STR_LIT>' % ( opts . out_file , i ) <EOL> if not os . path . exists ( fi ) or os . path . getsize ( fi ) == <NUM_LIT:0> : <EOL> wait_count += <NUM_LIT:1> <EOL> return wait_count <EOL> while jobs_left ( ) != <NUM_LIT:0> : <EOL> print '<STR_LIT:->' * <NUM_LIT> <EOL> print '<STR_LIT>' % jobs_left ( ) <EOL> print '<STR_LIT:->' * <NUM_LIT> <EOL> time . sleep ( SLEEP_SEC ) <EOL> concat_list = list ( ) <EOL> hyps = list ( ) <EOL> refs = list ( ) <EOL> hypscores = list ( ) <EOL> refscores = list ( ) <EOL> numphones = list ( ) <EOL> subsets = list ( ) <EOL> alignments = list ( ) <EOL> for i in xrange ( opts . start_file , opts . start_file + opts . numFiles ) : <EOL> fi = '<STR_LIT>' % ( opts . out_file , i ) <EOL> fi_pk = fi . replace ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> assert os . path . exists ( fi ) , '<STR_LIT>' % fi <EOL> assert os . path . exists ( fi_pk ) , '<STR_LIT>' % fi_pk <EOL> with open ( fi , '<STR_LIT:r>' ) as f : <EOL> concat_list . append ( f . read ( ) . strip ( ) ) <EOL> with open ( fi_pk , '<STR_LIT:rb>' ) as f : <EOL> h = pickle . load ( f ) <EOL> r = pickle . load ( f ) <EOL> hs = pickle . load ( f ) <EOL> rs = pickle . load ( f ) <EOL> np = pickle . load ( f ) <EOL> ss = pickle . load ( f ) <EOL> al = pickle . load ( f ) <EOL> hyps += h <EOL> refs += r <EOL> hypscores += hs <EOL> refscores += rs <EOL> numphones += np <EOL> subsets += ss <EOL> alignments += al <EOL> os . remove ( fi ) <EOL> os . remove ( fi_pk ) <EOL> with open ( opts . out_file , '<STR_LIT:w>' ) as fout : <EOL> fout . write ( '<STR_LIT:\n>' . join ( concat_list ) ) <EOL> with open ( opts . out_file . replace ( '<STR_LIT>' , '<STR_LIT>' ) , '<STR_LIT:wb>' ) as fout : <EOL> pickle . dump ( hyps , fout ) <EOL> pickle . dump ( refs , fout ) <EOL> pickle . dump ( hypscores , fout ) <EOL> pickle . dump ( refscores , fout ) <EOL> pickle . dump ( numphones , fout ) <EOL> pickle . dump ( subsets , fout ) <EOL> pickle . dump ( alignments , fout ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> import optparse <EOL> usage = "<STR_LIT>" <EOL> parser = optparse . OptionParser ( usage = usage ) <EOL> parser . add_option ( "<STR_LIT>" , dest = "<STR_LIT>" , type = "<STR_LIT:string>" , <EOL> default = "<STR_LIT>" ) <EOL> parser . add_option ( '<STR_LIT>' , dest = '<STR_LIT>' , type = '<STR_LIT:string>' , default = '<STR_LIT>' ) <EOL> parser . add_option ( "<STR_LIT>" , dest = "<STR_LIT>" , type = "<STR_LIT:int>" , default = <NUM_LIT> ) <EOL> parser . add_option ( <EOL> "<STR_LIT>" , dest = "<STR_LIT>" , type = "<STR_LIT:int>" , default = INPUT_DIM ) <EOL> parser . add_option ( "<STR_LIT>" , dest = "<STR_LIT>" , type = "<STR_LIT:int>" , default = RAW_DIM ) <EOL> parser . add_option ( '<STR_LIT>' , dest = '<STR_LIT>' , type = '<STR_LIT:string>' , default = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , dest = '<STR_LIT>' , type = '<STR_LIT:int>' , default = <NUM_LIT:1> ) <EOL> parser . add_option ( '<STR_LIT>' , action = '<STR_LIT:store_true>' , default = False , help = '<STR_LIT>' ) <EOL> ( opts , args ) = parser . parse_args ( ) <EOL> if opts . parallel : <EOL> runParallel ( opts ) <EOL> else : <EOL> runSeq ( opts ) </s>
