<s> import numbers <EOL> from itertools import product <EOL> from collections import defaultdict <EOL> import h5py <EOL> import numpy <EOL> import six <EOL> import tables <EOL> from six . moves import zip , range <EOL> from fuel . datasets import Dataset <EOL> from fuel . utils import do_not_pickle_attributes , Subset <EOL> from fuel . schemes import SequentialExampleScheme <EOL> @ do_not_pickle_attributes ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> class PytablesDataset ( Dataset ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , path , sources , start = <NUM_LIT:0> , stop = None , data_node = '<STR_LIT>' , <EOL> sources_in_file = None ) : <EOL> if sources_in_file is None : <EOL> sources_in_file = sources <EOL> self . sources_in_file = sources_in_file <EOL> self . provides_sources = sources <EOL> self . path = path <EOL> self . data_node = data_node <EOL> self . start = start <EOL> self . stop = stop <EOL> self . nodes = None <EOL> self . open_file ( path ) <EOL> super ( PytablesDataset , self ) . __init__ ( self . provides_sources ) <EOL> def open_file ( self , path ) : <EOL> self . h5file = tables . open_file ( path , mode = "<STR_LIT:r>" ) <EOL> node = self . h5file . get_node ( '<STR_LIT:/>' , self . data_node ) <EOL> self . nodes = [ getattr ( node , source ) for source in self . sources_in_file ] <EOL> if self . stop is None : <EOL> self . stop = self . nodes [ <NUM_LIT:0> ] . nrows <EOL> self . num_examples = self . stop - self . start <EOL> def load ( self ) : <EOL> self . open_file ( self . path ) <EOL> def close_file ( self ) : <EOL> self . h5file . close ( ) <EOL> del self . _h5file <EOL> del self . _nodes <EOL> def get_data ( self , state = None , request = None ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( request , slice ) : <EOL> request = slice ( request . start + self . start , <EOL> request . stop + self . start , request . step ) <EOL> data = [ node [ request ] for node in self . nodes ] <EOL> elif isinstance ( request , list ) : <EOL> request = [ index + self . start for index in request ] <EOL> data = [ node [ request , ... ] for node in self . nodes ] <EOL> else : <EOL> raise ValueError <EOL> return data <EOL> @ do_not_pickle_attributes ( '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> class H5PYDataset ( Dataset ) : <EOL> """<STR_LIT>""" <EOL> interface_version = '<STR_LIT>' <EOL> _ref_counts = defaultdict ( int ) <EOL> _file_handles = { } <EOL> def __init__ ( self , file_or_path , which_sets , subset = None , <EOL> load_in_memory = False , driver = None , sort_indices = True , <EOL> ** kwargs ) : <EOL> if isinstance ( file_or_path , h5py . File ) : <EOL> self . path = file_or_path . filename <EOL> self . external_file_handle = file_or_path <EOL> else : <EOL> self . path = file_or_path <EOL> self . external_file_handle = None <EOL> which_sets_invalid_value = ( <EOL> isinstance ( which_sets , six . string_types ) or <EOL> not all ( isinstance ( s , six . string_types ) for s in which_sets ) ) <EOL> if which_sets_invalid_value : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> self . which_sets = which_sets <EOL> self . user_given_subset = subset if subset else slice ( None ) <EOL> self . load_in_memory = load_in_memory <EOL> self . driver = driver <EOL> self . sort_indices = sort_indices <EOL> self . _parse_dataset_info ( ) <EOL> kwargs . setdefault ( '<STR_LIT>' , self . default_axis_labels ) <EOL> super ( H5PYDataset , self ) . __init__ ( ** kwargs ) <EOL> self . example_iteration_scheme = SequentialExampleScheme ( <EOL> self . num_examples ) <EOL> def _parse_dataset_info ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _out_of_memory_open ( ) <EOL> handle = self . _file_handle <EOL> available_splits = self . get_all_splits ( handle ) <EOL> which_sets = self . which_sets <EOL> provides_sources = None <EOL> for split in which_sets : <EOL> if split not in available_splits : <EOL> raise ValueError ( <EOL> "<STR_LIT>" . format ( split ) + <EOL> "<STR_LIT>" + <EOL> "<STR_LIT>" . format ( available_splits ) ) <EOL> split_provides_sources = set ( <EOL> self . get_provided_sources ( handle , split ) ) <EOL> if provides_sources : <EOL> provides_sources &= split_provides_sources <EOL> else : <EOL> provides_sources = split_provides_sources <EOL> self . provides_sources = tuple ( sorted ( provides_sources ) ) <EOL> self . vlen_sources = self . get_vlen_sources ( handle ) <EOL> self . default_axis_labels = self . get_axis_labels ( handle ) <EOL> self . _out_of_memory_close ( ) <EOL> @ staticmethod <EOL> def create_split_array ( split_dict ) : <EOL> """<STR_LIT>""" <EOL> split_len = max ( len ( split ) for split in split_dict ) <EOL> sources = set ( ) <EOL> comment_len = <NUM_LIT:1> <EOL> for split in split_dict . values ( ) : <EOL> sources |= set ( split . keys ( ) ) <EOL> for val in split . values ( ) : <EOL> if len ( val ) == <NUM_LIT:4> : <EOL> comment_len = max ( [ comment_len , len ( val [ - <NUM_LIT:1> ] ) ] ) <EOL> sources = sorted ( list ( sources ) ) <EOL> source_len = max ( len ( source ) for source in sources ) <EOL> split_array = numpy . empty ( <EOL> len ( split_dict ) * len ( sources ) , <EOL> dtype = numpy . dtype ( [ <EOL> ( '<STR_LIT>' , '<STR_LIT:a>' , split_len ) , <EOL> ( '<STR_LIT:source>' , '<STR_LIT:a>' , source_len ) , <EOL> ( '<STR_LIT:start>' , numpy . int64 , <NUM_LIT:1> ) , <EOL> ( '<STR_LIT>' , numpy . int64 , <NUM_LIT:1> ) , <EOL> ( '<STR_LIT>' , h5py . special_dtype ( ref = h5py . Reference ) ) , <EOL> ( '<STR_LIT>' , numpy . bool , <NUM_LIT:1> ) , <EOL> ( '<STR_LIT>' , '<STR_LIT:a>' , comment_len ) ] ) ) <EOL> for i , ( split , source ) in enumerate ( product ( split_dict , sources ) ) : <EOL> if source in split_dict [ split ] : <EOL> start , stop = split_dict [ split ] [ source ] [ : <NUM_LIT:2> ] <EOL> available = True <EOL> indices = h5py . Reference ( ) <EOL> comment = '<STR_LIT:.>' <EOL> if len ( split_dict [ split ] [ source ] ) > <NUM_LIT:2> : <EOL> indices = split_dict [ split ] [ source ] [ <NUM_LIT:2> ] <EOL> if len ( split_dict [ split ] [ source ] ) > <NUM_LIT:3> : <EOL> comment = split_dict [ split ] [ source ] [ <NUM_LIT:3> ] <EOL> if not comment : <EOL> comment = '<STR_LIT:.>' <EOL> else : <EOL> ( start , stop , indices , available , comment ) = ( <EOL> <NUM_LIT:0> , <NUM_LIT:0> , h5py . Reference ( ) , False , '<STR_LIT:.>' ) <EOL> split_array [ i ] [ '<STR_LIT>' ] = split . encode ( '<STR_LIT:utf8>' ) <EOL> split_array [ i ] [ '<STR_LIT:source>' ] = source . encode ( '<STR_LIT:utf8>' ) <EOL> split_array [ i ] [ '<STR_LIT:start>' ] = start <EOL> split_array [ i ] [ '<STR_LIT>' ] = stop <EOL> split_array [ i ] [ '<STR_LIT>' ] = indices <EOL> split_array [ i ] [ '<STR_LIT>' ] = available <EOL> split_array [ i ] [ '<STR_LIT>' ] = comment . encode ( '<STR_LIT:utf8>' ) <EOL> return split_array <EOL> @ staticmethod <EOL> def get_all_splits ( h5file ) : <EOL> """<STR_LIT>""" <EOL> available_splits = tuple ( <EOL> set ( row [ '<STR_LIT>' ] . decode ( '<STR_LIT:utf8>' ) for row in h5file . attrs [ '<STR_LIT>' ] ) ) <EOL> return available_splits <EOL> @ staticmethod <EOL> def get_all_sources ( h5file ) : <EOL> """<STR_LIT>""" <EOL> all_sources = tuple ( <EOL> set ( row [ '<STR_LIT:source>' ] . decode ( '<STR_LIT:utf8>' ) for row in h5file . attrs [ '<STR_LIT>' ] ) ) <EOL> return all_sources <EOL> @ staticmethod <EOL> def get_provided_sources ( h5file , split ) : <EOL> """<STR_LIT>""" <EOL> provided_sources = tuple ( <EOL> row [ '<STR_LIT:source>' ] . decode ( '<STR_LIT:utf8>' ) for row in h5file . attrs [ '<STR_LIT>' ] <EOL> if row [ '<STR_LIT>' ] . decode ( '<STR_LIT:utf8>' ) == split and row [ '<STR_LIT>' ] ) <EOL> return provided_sources <EOL> @ staticmethod <EOL> def get_vlen_sources ( h5file ) : <EOL> """<STR_LIT>""" <EOL> vlen_sources = [ ] <EOL> for source_name in H5PYDataset . get_all_sources ( h5file ) : <EOL> source = h5file [ source_name ] <EOL> if len ( source . dims ) > <NUM_LIT:0> and '<STR_LIT>' in source . dims [ <NUM_LIT:0> ] : <EOL> if len ( source . dims ) > <NUM_LIT:1> : <EOL> raise ValueError ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> vlen_sources . append ( source_name ) <EOL> return vlen_sources <EOL> @ staticmethod <EOL> def get_axis_labels ( h5file ) : <EOL> """<STR_LIT>""" <EOL> axis_labels = { } <EOL> vlen_sources = H5PYDataset . get_vlen_sources ( h5file ) <EOL> for source_name in H5PYDataset . get_all_sources ( h5file ) : <EOL> if source_name in vlen_sources : <EOL> axis_labels [ source_name ] = ( <EOL> ( h5file [ source_name ] . dims [ <NUM_LIT:0> ] . label , ) + <EOL> tuple ( label . decode ( '<STR_LIT:utf8>' ) for label in <EOL> h5file [ source_name ] . dims [ <NUM_LIT:0> ] [ '<STR_LIT>' ] ) ) <EOL> else : <EOL> axis_labels [ source_name ] = tuple ( <EOL> dim . label for dim in h5file [ source_name ] . dims ) <EOL> return axis_labels <EOL> @ staticmethod <EOL> def get_subsets ( h5file , splits , sources ) : <EOL> """<STR_LIT>""" <EOL> subsets = [ Subset . empty_subset ( len ( h5file [ source_name ] ) ) <EOL> for source_name in sources ] <EOL> for split in splits : <EOL> for i , source in enumerate ( sources ) : <EOL> row , = [ r for r in h5file . attrs [ '<STR_LIT>' ] if <EOL> ( r [ '<STR_LIT>' ] . decode ( '<STR_LIT:utf8>' ) == split and <EOL> r [ '<STR_LIT:source>' ] . decode ( '<STR_LIT:utf8>' ) == source ) ] <EOL> if row [ '<STR_LIT>' ] : <EOL> subsets [ i ] += Subset ( <EOL> h5file [ row [ '<STR_LIT>' ] ] , len ( h5file [ source ] ) ) <EOL> else : <EOL> subsets [ i ] += Subset ( <EOL> slice ( row [ '<STR_LIT:start>' ] , row [ '<STR_LIT>' ] ) , len ( h5file [ source ] ) ) <EOL> return subsets <EOL> def load ( self ) : <EOL> if not hasattr ( self , '<STR_LIT>' ) : <EOL> self . external_file_handle = None <EOL> self . _out_of_memory_open ( ) <EOL> handle = self . _file_handle <EOL> subsets = self . get_subsets ( handle , self . which_sets , self . sources ) <EOL> if any ( subset . num_examples != subsets [ <NUM_LIT:0> ] . num_examples for subset in <EOL> subsets ) : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> self . subsets = [ Subset . subset_of ( subset , self . user_given_subset ) <EOL> for subset in subsets ] <EOL> if self . load_in_memory : <EOL> data_sources = [ ] <EOL> source_shapes = [ ] <EOL> for source_name , subset in zip ( self . sources , self . subsets ) : <EOL> data_sources . append ( <EOL> subset . index_within_subset ( <EOL> handle [ source_name ] , slice ( None ) ) ) <EOL> if source_name in self . vlen_sources : <EOL> shapes = subset . index_within_subset ( <EOL> handle [ source_name ] . dims [ <NUM_LIT:0> ] [ '<STR_LIT>' ] , <EOL> slice ( None ) ) <EOL> else : <EOL> shapes = None <EOL> source_shapes . append ( shapes ) <EOL> self . data_sources = tuple ( data_sources ) <EOL> self . source_shapes = tuple ( source_shapes ) <EOL> self . in_memory_subset = Subset ( <EOL> slice ( None ) , len ( self . data_sources [ <NUM_LIT:0> ] ) ) <EOL> else : <EOL> self . data_sources = None <EOL> self . source_shapes = None <EOL> self . in_memory_subset = None <EOL> self . _out_of_memory_close ( ) <EOL> @ property <EOL> def num_examples ( self ) : <EOL> return self . subsets [ <NUM_LIT:0> ] . num_examples <EOL> def open ( self ) : <EOL> return None if self . load_in_memory else self . _out_of_memory_open ( ) <EOL> def _out_of_memory_open ( self ) : <EOL> if not self . external_file_handle : <EOL> if self . path not in self . _file_handles : <EOL> handle = h5py . File ( <EOL> name = self . path , mode = "<STR_LIT:r>" , driver = self . driver ) <EOL> self . _file_handles [ self . path ] = handle <EOL> self . _ref_counts [ self . path ] += <NUM_LIT:1> <EOL> def close ( self , state ) : <EOL> if not self . load_in_memory : <EOL> self . _out_of_memory_close ( ) <EOL> def _out_of_memory_close ( self ) : <EOL> if not self . external_file_handle : <EOL> self . _ref_counts [ self . path ] -= <NUM_LIT:1> <EOL> if not self . _ref_counts [ self . path ] : <EOL> del self . _ref_counts [ self . path ] <EOL> self . _file_handles [ self . path ] . close ( ) <EOL> del self . _file_handles [ self . path ] <EOL> @ property <EOL> def _file_handle ( self ) : <EOL> if self . external_file_handle : <EOL> return self . external_file_handle <EOL> elif self . path in self . _file_handles : <EOL> return self . _file_handles [ self . path ] <EOL> else : <EOL> raise IOError ( '<STR_LIT>' . format ( self . path ) ) <EOL> def get_data ( self , state = None , request = None ) : <EOL> if self . load_in_memory : <EOL> data , shapes = self . _in_memory_get_data ( state , request ) <EOL> else : <EOL> data , shapes = self . _out_of_memory_get_data ( state , request ) <EOL> for i in range ( len ( data ) ) : <EOL> if shapes [ i ] is not None : <EOL> if isinstance ( request , numbers . Integral ) : <EOL> data [ i ] = data [ i ] . reshape ( shapes [ i ] ) <EOL> else : <EOL> for j in range ( len ( data [ i ] ) ) : <EOL> data [ i ] [ j ] = data [ i ] [ j ] . reshape ( shapes [ i ] [ j ] ) <EOL> return tuple ( data ) <EOL> def _in_memory_get_data ( self , state = None , request = None ) : <EOL> if state is not None or request is None : <EOL> raise ValueError <EOL> data = [ self . in_memory_subset . index_within_subset ( data_source , request ) <EOL> for data_source in self . data_sources ] <EOL> shapes = [ self . in_memory_subset . index_within_subset ( shape , request ) <EOL> if shape is not None else None <EOL> for shape in self . source_shapes ] <EOL> return data , shapes <EOL> def _out_of_memory_get_data ( self , state = None , request = None ) : <EOL> if not isinstance ( request , ( numbers . Integral , slice , list ) ) : <EOL> raise ValueError ( ) <EOL> data = [ ] <EOL> shapes = [ ] <EOL> handle = self . _file_handle <EOL> for source_name , subset in zip ( self . sources , self . subsets ) : <EOL> data . append ( <EOL> subset . index_within_subset ( <EOL> handle [ source_name ] , request , <EOL> sort_indices = self . sort_indices ) ) <EOL> if source_name in self . vlen_sources : <EOL> shapes . append ( <EOL> subset . index_within_subset ( <EOL> handle [ source_name ] . dims [ <NUM_LIT:0> ] [ '<STR_LIT>' ] , request , <EOL> sort_indices = self . sort_indices ) ) <EOL> else : <EOL> shapes . append ( None ) <EOL> return data , shapes </s>
