<s> """<STR_LIT>""" <EOL> __authors__ = [ "<STR_LIT>" ] <EOL> __copyright__ = "<STR_LIT>" <EOL> __credits__ = [ "<STR_LIT>" , "<STR_LIT>" ] <EOL> __license__ = "<STR_LIT>" <EOL> __maintainer__ = "<STR_LIT>" <EOL> __email__ = "<STR_LIT>" <EOL> import gc <EOL> import os . path <EOL> import functools <EOL> import numpy <EOL> from pylearn2 . utils . iteration import FiniteDatasetIterator , resolve_iterator_class <EOL> from pylearn2 . datasets . dataset import Dataset <EOL> from pylearn2 . space import CompositeSpace , VectorSpace , IndexSpace <EOL> from pylearn2 . utils import serial <EOL> from pylearn2 . utils import safe_zip <EOL> from segmentaxis import segment_axis <EOL> import scipy . stats <EOL> from scikits . talkbox import lpc <EOL> class TIMITlpc ( Dataset ) : <EOL> """<STR_LIT>""" <EOL> _default_seed = ( <NUM_LIT> , <NUM_LIT:2> , <NUM_LIT> ) <EOL> _mean = <NUM_LIT> <EOL> _std = <NUM_LIT> <EOL> def __init__ ( self , which_set , frame_length , overlap = <NUM_LIT:0> , lpc_order = <NUM_LIT:10> , <EOL> frames_per_example = <NUM_LIT:1> , start = <NUM_LIT:0> , stop = None , audio_only = False , <EOL> rng = _default_seed ) : <EOL> """<STR_LIT>""" <EOL> self . frame_length = frame_length <EOL> self . overlap = overlap <EOL> self . frames_per_example = frames_per_example <EOL> self . lpc_order = lpc_order <EOL> self . offset = self . frame_length - self . overlap <EOL> self . audio_only = audio_only <EOL> if hasattr ( rng , '<STR_LIT>' ) : <EOL> self . rng = rng <EOL> else : <EOL> self . rng = numpy . random . RandomState ( rng ) <EOL> self . _load_data ( which_set ) <EOL> if stop is not None : <EOL> self . raw_wav = self . raw_wav [ start : stop ] <EOL> if not self . audio_only : <EOL> self . phones = self . phones [ start : stop ] <EOL> self . phonemes = self . phonemes [ start : stop ] <EOL> self . words = self . words [ start : stop ] <EOL> else : <EOL> self . raw_wav = self . raw_wav [ start : ] <EOL> if not self . audio_only : <EOL> self . phones = self . phones [ start : ] <EOL> self . phonemes = self . phonemes [ start : ] <EOL> self . words = self . words [ start : ] <EOL> examples_per_sequence = [ <NUM_LIT:0> ] <EOL> for sequence_id , samples_sequence in enumerate ( self . raw_wav ) : <EOL> if not self . audio_only : <EOL> phones_sequence = self . phones [ sequence_id ] <EOL> phones_segmented_sequence = segment_axis ( phones_sequence , <EOL> frame_length , <EOL> overlap ) <EOL> self . phones [ sequence_id ] = phones_segmented_sequence <EOL> phonemes_sequence = self . phonemes [ sequence_id ] <EOL> phonemes_segmented_sequence = segment_axis ( phonemes_sequence , <EOL> frame_length , <EOL> overlap ) <EOL> self . phonemes [ sequence_id ] = phonemes_segmented_sequence <EOL> words_sequence = self . words [ sequence_id ] <EOL> words_segmented_sequence = segment_axis ( words_sequence , <EOL> frame_length , <EOL> overlap ) <EOL> self . words [ sequence_id ] = words_segmented_sequence <EOL> samples_segmented_sequence = segment_axis ( samples_sequence , <EOL> frame_length , <EOL> overlap ) <EOL> self . raw_wav [ sequence_id ] = samples_segmented_sequence <EOL> num_frames = samples_segmented_sequence . shape [ <NUM_LIT:0> ] <EOL> num_examples = num_frames - self . frames_per_example <EOL> examples_per_sequence . append ( num_examples ) <EOL> self . cumulative_example_indexes = numpy . cumsum ( examples_per_sequence ) <EOL> self . samples_sequences = self . raw_wav <EOL> if not self . audio_only : <EOL> self . phones_sequences = self . phones <EOL> self . phonemes_sequences = self . phonemes <EOL> self . words_sequences = self . words <EOL> self . num_examples = self . cumulative_example_indexes [ - <NUM_LIT:1> ] <EOL> targets_space = VectorSpace ( <EOL> dim = self . lpc_order <EOL> ) <EOL> targets_source = '<STR_LIT>' <EOL> targets_dtype = numpy . float64 <EOL> def targets_map_fn ( indexes ) : <EOL> rval = [ ] <EOL> for sequence_index , example_index in self . _fetch_index ( indexes ) : <EOL> rval . append ( lpc ( self . samples_sequences [ sequence_index ] [ example_index ] . T , self . lpc_order ) [ <NUM_LIT:0> ] [ <NUM_LIT:1> : ] . ravel ( ) ) <EOL> return rval <EOL> features_space = IndexSpace ( max_labels = <NUM_LIT> , dim = <NUM_LIT:3> ) <EOL> features_source = '<STR_LIT>' <EOL> features_dtype = numpy . int16 <EOL> def features_map_fn ( indexes ) : <EOL> rval = [ ] <EOL> for sequence_index , example_index in self . _fetch_index ( indexes ) : <EOL> if example_index != <NUM_LIT:0> : <EOL> if example_index != self . num_examples - <NUM_LIT:1> : <EOL> rval . append ( numpy . asarray ( scipy . stats . mode ( self . phones_sequences [ sequence_index ] [ ( example_index - <NUM_LIT:1> ) : ( example_index + <NUM_LIT:2> ) ] , axis = <NUM_LIT:1> ) [ <NUM_LIT:0> ] . T , dtype = numpy . int16 ) ) <EOL> else : <EOL> rval . append ( numpy . asarray ( numpy . concatenate ( [ scipy . stats . mode ( self . phones_sequences [ sequence_index ] [ example_index - <NUM_LIT:1> : example_index + <NUM_LIT:1> ] , axis = <NUM_LIT:1> ) [ <NUM_LIT:0> ] . T , numpy . zeros ( ( <NUM_LIT:1> , <NUM_LIT:1> ) ) ] , axis = <NUM_LIT:1> ) . ravel ( ) , dtype = numpy . int16 ) ) <EOL> else : <EOL> rval . append ( numpy . asarray ( numpy . concatenate ( [ numpy . zeros ( ( <NUM_LIT:1> , <NUM_LIT:1> ) ) , scipy . stats . mode ( self . phones_sequences [ sequence_index ] [ example_index : example_index + <NUM_LIT:2> ] . T ) [ <NUM_LIT:0> ] . T ] ) . ravel ( ) , dtype = numpy . int16 ) ) <EOL> return rval <EOL> space_components = [ features_space , targets_space ] <EOL> source_components = [ features_source , targets_source ] <EOL> dtypes_components = [ features_dtype , targets_dtype ] <EOL> map_fn_components = [ features_map_fn , targets_map_fn ] <EOL> batch_components = [ None , None ] <EOL> space = CompositeSpace ( space_components ) <EOL> source = tuple ( source_components ) <EOL> self . data_specs = ( space , source ) <EOL> self . dtypes = tuple ( dtypes_components ) <EOL> self . map_functions = tuple ( map_fn_components ) <EOL> self . batch_buffers = batch_components <EOL> self . _iter_mode = resolve_iterator_class ( '<STR_LIT>' ) <EOL> self . _iter_data_specs = ( CompositeSpace ( ( features_space , <EOL> targets_space ) ) , <EOL> ( features_source , targets_source ) ) <EOL> def _fetch_index ( self , indexes ) : <EOL> digit = numpy . digitize ( indexes , self . cumulative_example_indexes ) - <NUM_LIT:1> <EOL> return zip ( digit , <EOL> numpy . array ( indexes ) - self . cumulative_example_indexes [ digit ] ) <EOL> def _load_data ( self , which_set ) : <EOL> """<STR_LIT>""" <EOL> if which_set not in [ '<STR_LIT:train>' , '<STR_LIT>' , '<STR_LIT:test>' ] : <EOL> raise ValueError ( which_set + "<STR_LIT>" + <EOL> "<STR_LIT>" ) <EOL> timit_base_path = os . path . join ( os . environ [ "<STR_LIT>" ] , <EOL> "<STR_LIT>" ) <EOL> speaker_info_list_path = os . path . join ( timit_base_path , "<STR_LIT>" ) <EOL> phonemes_list_path = os . path . join ( timit_base_path , <EOL> "<STR_LIT>" ) <EOL> words_list_path = os . path . join ( timit_base_path , "<STR_LIT>" ) <EOL> speaker_features_list_path = os . path . join ( timit_base_path , <EOL> "<STR_LIT>" ) <EOL> speaker_id_list_path = os . path . join ( timit_base_path , <EOL> "<STR_LIT>" ) <EOL> raw_wav_path = os . path . join ( timit_base_path , which_set + "<STR_LIT>" ) <EOL> phonemes_path = os . path . join ( timit_base_path , <EOL> which_set + "<STR_LIT>" ) <EOL> phones_path = os . path . join ( timit_base_path , <EOL> which_set + "<STR_LIT>" ) <EOL> words_path = os . path . join ( timit_base_path , which_set + "<STR_LIT>" ) <EOL> speaker_path = os . path . join ( timit_base_path , <EOL> which_set + "<STR_LIT>" ) <EOL> if not self . audio_only : <EOL> self . speaker_info_list = serial . load ( <EOL> speaker_info_list_path <EOL> ) . tolist ( ) . toarray ( ) <EOL> self . speaker_id_list = serial . load ( speaker_id_list_path ) <EOL> self . speaker_features_list = serial . load ( speaker_features_list_path ) <EOL> self . words_list = serial . load ( words_list_path ) <EOL> self . phonemes_list = serial . load ( phonemes_list_path ) <EOL> self . raw_wav = serial . load ( raw_wav_path ) <EOL> if not self . audio_only : <EOL> self . phonemes = serial . load ( phonemes_path ) <EOL> self . phones = serial . load ( phones_path ) <EOL> self . words = serial . load ( words_path ) <EOL> self . speaker_id = numpy . asarray ( serial . load ( speaker_path ) , '<STR_LIT:int>' ) <EOL> def _validate_source ( self , source ) : <EOL> """<STR_LIT>""" <EOL> for s in source : <EOL> try : <EOL> self . data_specs [ <NUM_LIT:1> ] . index ( s ) <EOL> except ValueError : <EOL> raise ValueError ( "<STR_LIT>" + s + "<STR_LIT>" + <EOL> "<STR_LIT>" ) <EOL> def dtype_of ( self , source ) : <EOL> """<STR_LIT>""" <EOL> self . _validate_source ( source ) <EOL> return tuple ( self . dtypes [ self . data_specs [ <NUM_LIT:1> ] . index ( so ) ] <EOL> for so in source ) <EOL> def get_data_specs ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . data_specs <EOL> def get ( self , source , indexes ) : <EOL> """<STR_LIT>""" <EOL> if type ( indexes ) is slice : <EOL> indexes = numpy . arange ( indexes . start , indexes . stop ) <EOL> self . _validate_source ( source ) <EOL> rval = [ ] <EOL> for so in source : <EOL> batch = self . map_functions [ self . data_specs [ <NUM_LIT:1> ] . index ( so ) ] ( indexes ) <EOL> batch_buffer = self . batch_buffers [ self . data_specs [ <NUM_LIT:1> ] . index ( so ) ] <EOL> dim = self . data_specs [ <NUM_LIT:0> ] . components [ self . data_specs [ <NUM_LIT:1> ] . index ( so ) ] . dim <EOL> if batch_buffer is None or batch_buffer . shape != ( len ( batch ) , dim ) : <EOL> batch_buffer = numpy . zeros ( ( len ( batch ) , dim ) , <EOL> dtype = batch [ <NUM_LIT:0> ] . dtype ) <EOL> for i , example in enumerate ( batch ) : <EOL> batch_buffer [ i ] = example <EOL> rval . append ( batch_buffer ) <EOL> return tuple ( rval ) <EOL> @ functools . wraps ( Dataset . iterator ) <EOL> def iterator ( self , mode = None , batch_size = None , num_batches = None , <EOL> rng = None , data_specs = None , return_tuple = False , convert = None ) : <EOL> """<STR_LIT>""" <EOL> if data_specs is None : <EOL> data_specs = self . _iter_data_specs <EOL> space , source = data_specs <EOL> if isinstance ( space , CompositeSpace ) : <EOL> sub_spaces = space . components <EOL> sub_sources = source <EOL> else : <EOL> sub_spaces = ( space , ) <EOL> sub_sources = ( source , ) <EOL> if convert == None : <EOL> if hasattr ( self , '<STR_LIT>' ) : <EOL> convert = self . _iter_convert <EOL> else : <EOL> convert = [ ] <EOL> for sp , src in safe_zip ( sub_spaces , sub_sources ) : <EOL> convert . append ( None ) <EOL> if mode is None : <EOL> if hasattr ( self , '<STR_LIT>' ) : <EOL> mode = self . _iter_subset_class <EOL> else : <EOL> raise ValueError ( '<STR_LIT>' <EOL> '<STR_LIT>' % str ( self ) ) <EOL> else : <EOL> mode = resolve_iterator_class ( mode ) <EOL> if batch_size is None : <EOL> batch_size = getattr ( self , '<STR_LIT>' , None ) <EOL> if num_batches is None : <EOL> num_batches = getattr ( self , '<STR_LIT>' , None ) <EOL> if rng is None and mode . stochastic : <EOL> rng = self . rng <EOL> return FiniteDatasetIterator ( self , <EOL> mode ( self . num_examples , batch_size , <EOL> num_batches , rng ) , <EOL> data_specs = data_specs , <EOL> return_tuple = return_tuple , <EOL> convert = convert ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> valid_timit = TIMIT ( "<STR_LIT>" , frame_length = <NUM_LIT:1> , overlap = <NUM_LIT:0> , <EOL> frames_per_example = <NUM_LIT:100> , audio_only = False ) <EOL> it = valid_timit . iterator ( mode = '<STR_LIT>' , num_batches = <NUM_LIT:100> , batch_size = <NUM_LIT> ) <EOL> import pdb ; pdb . set_trace ( ) <EOL> for ( f , t ) in it : <EOL> print f . shape </s>
