pypet/storageservice.py
def _prm_write_into_pytable(self, tablename, data, hdf5_group, fullname, **kwargs):
        """Stores data as pytable.

        :param tablename:

            Name of the data table

        :param data:

            Data to store

        :param hdf5_group:

            Group node where to store data in hdf5 file

        :param fullname:

            Full name of the `data_to_store`s original container, only needed for throwing errors.

        """

        datasize = data.shape[0]

        try:

            # Get a new pytables description from the data and create a new table
            description_dict, data_type_dict = self._prm_make_description(data, fullname)
            description_dicts = [{}]

            if len(description_dict) > ptpa.MAX_COLUMNS:
                # For optimization we want to store the original data types into another table
                # and split the tables into several ones
                new_table_group = self._hdf5file.create_group(where=hdf5_group,
                                                    name=tablename,
                                                    filters=self._all_get_filters(kwargs.copy()))

                count = 0
                for innerkey in description_dict:
                    val = description_dict[innerkey]
                    if count == ptpa.MAX_COLUMNS:
                        description_dicts.append({})
                        count = 0
                    description_dicts[-1][innerkey] = val
                    count += 1

                setattr(new_table_group._v_attrs, HDF5StorageService.STORAGE_TYPE,
                        HDF5StorageService.TABLE)
                setattr(new_table_group._v_attrs, HDF5StorageService.SPLIT_TABLE, 1)

                hdf5_group = new_table_group
            else:
                description_dicts = [description_dict]

            for idx, descr_dict in enumerate(description_dicts):

                if idx == 0:
                    tblname = tablename
                else:
                    tblname = tablename + '_%d' % idx

                table = self._hdf5file.create_table(where=hdf5_group, name=tblname,
                                              description=descr_dict,
                                              title=tblname,
                                              expectedrows=datasize,
                                              filters=self._all_get_filters(kwargs.copy()))

                row = table.row
                for n in range(datasize):
                    # Fill the columns with data, note if the parameter was extended nstart!=0

                    for key in descr_dict:
                        row[key] = data[key][n]

                    row.append()

                # Remember the original types of the data for perfect recall
                if idx == 0 and len(description_dict) <= ptpa.MAX_COLUMNS:
                    # We only have a single table and
                    # we can store the original data types as attributes
                    for field_name in data_type_dict:
                        type_description = data_type_dict[field_name]
                        self._all_set_attr(table, field_name, type_description)

                    setattr(table._v_attrs, HDF5StorageService.STORAGE_TYPE,
                            HDF5StorageService.TABLE)

                table.flush()
                self._hdf5file.flush()

            if len(description_dict) > ptpa.MAX_COLUMNS:
                # We have potentially many split tables and the data types are
                # stored into an additional table for performance reasons
                tblname = tablename + '__' + HDF5StorageService.STORAGE_TYPE
                field_names, data_types = list(zip(*data_type_dict.items()))
                data_type_table_dict = {'field_name': field_names, 'data_type': data_types}
                descr_dict, _ = self._prm_make_description(data_type_table_dict, fullname)

                table = self._hdf5file.create_table(where=hdf5_group, name=tblname,
                                              description=descr_dict,
                                              title=tblname,
                                              expectedrows=len(field_names),
                                              filters=self._all_get_filters(kwargs))

                row = table.row

                for n in range(len(field_names)):
                    # Fill the columns with data

                    for key in data_type_table_dict:
                        row[key] = data_type_table_dict[key][n]

                    row.append()

                setattr(table._v_attrs, HDF5StorageService.DATATYPE_TABLE, 1)

                table.flush()
                self._hdf5file.flush()

        except:
            self._logger.error('Failed storing table `%s` of `%s`.' % (tablename, fullname))
            raise
def _prm_write_into_pytable(self, tablename, data, hdf5_group, fullname, **kwargs):
        datasize = data.shape[0]

        try:

            # Get a new pytables description from the data and create a new table
            description_dict, data_type_dict = self._prm_make_description(data, fullname)
            description_dicts = [{}]

            if len(description_dict) > ptpa.MAX_COLUMNS:
                # For optimization we want to store the original data types into another table
                # and split the tables into several ones
                new_table_group = self._hdf5file.create_group(where=hdf5_group,
                                                    name=tablename,
                                                    filters=self._all_get_filters(kwargs.copy()))

                count = 0
                for innerkey in description_dict:
                    val = description_dict[innerkey]
                    if count == ptpa.MAX_COLUMNS:
                        description_dicts.append({})
                        count = 0
                    description_dicts[-1][innerkey] = val
                    count += 1

                setattr(new_table_group._v_attrs, HDF5StorageService.STORAGE_TYPE,
                        HDF5StorageService.TABLE)
                setattr(new_table_group._v_attrs, HDF5StorageService.SPLIT_TABLE, 1)

                hdf5_group = new_table_group
            else:
                description_dicts = [description_dict]

            for idx, descr_dict in enumerate(description_dicts):

                if idx == 0:
                    tblname = tablename
                else:
                    tblname = tablename + '_%d' % idx

                table = self._hdf5file.create_table(where=hdf5_group, name=tblname,
                                              description=descr_dict,
                                              title=tblname,
                                              expectedrows=datasize,
                                              filters=self._all_get_filters(kwargs.copy()))

                row = table.row
                for n in range(datasize):
                    # Fill the columns with data, note if the parameter was extended nstart!=0

                    for key in descr_dict:
                        row[key] = data[key][n]

                    row.append()

                # Remember the original types of the data for perfect recall
                if idx == 0 and len(description_dict) <= ptpa.MAX_COLUMNS:
                    # We only have a single table and
                    # we can store the original data types as attributes
                    for field_name in data_type_dict:
                        type_description = data_type_dict[field_name]
                        self._all_set_attr(table, field_name, type_description)

                    setattr(table._v_attrs, HDF5StorageService.STORAGE_TYPE,
                            HDF5StorageService.TABLE)

                table.flush()
                self._hdf5file.flush()

            if len(description_dict) > ptpa.MAX_COLUMNS:
                # We have potentially many split tables and the data types are
                # stored into an additional table for performance reasons
                tblname = tablename + '__' + HDF5StorageService.STORAGE_TYPE
                field_names, data_types = list(zip(*data_type_dict.items()))
                data_type_table_dict = {'field_name': field_names, 'data_type': data_types}
                descr_dict, _ = self._prm_make_description(data_type_table_dict, fullname)

                table = self._hdf5file.create_table(where=hdf5_group, name=tblname,
                                              description=descr_dict,
                                              title=tblname,
                                              expectedrows=len(field_names),
                                              filters=self._all_get_filters(kwargs))

                row = table.row

                for n in range(len(field_names)):
                    # Fill the columns with data

                    for key in data_type_table_dict:
                        row[key] = data_type_table_dict[key][n]

                    row.append()

                setattr(table._v_attrs, HDF5StorageService.DATATYPE_TABLE, 1)

                table.flush()
                self._hdf5file.flush()

        except:
            self._logger.error('Failed storing table `%s` of `%s`.' % (tablename, fullname))
            raise
def _prm_write_into_pytable ( self , tablename , data , hdf5_group , fullname , * * kwargs ) : datasize = data . shape [ 0 ] try : # Get a new pytables description from the data and create a new table description_dict , data_type_dict = self . _prm_make_description ( data , fullname ) description_dicts = [ { } ] if len ( description_dict ) > ptpa . MAX_COLUMNS : # For optimization we want to store the original data types into another table # and split the tables into several ones new_table_group = self . _hdf5file . create_group ( where = hdf5_group , name = tablename , filters = self . _all_get_filters ( kwargs . copy ( ) ) ) count = 0 for innerkey in description_dict : val = description_dict [ innerkey ] if count == ptpa . MAX_COLUMNS : description_dicts . append ( { } ) count = 0 description_dicts [ - 1 ] [ innerkey ] = val count += 1 setattr ( new_table_group . _v_attrs , HDF5StorageService . STORAGE_TYPE , HDF5StorageService . TABLE ) setattr ( new_table_group . _v_attrs , HDF5StorageService . SPLIT_TABLE , 1 ) hdf5_group = new_table_group else : description_dicts = [ description_dict ] for idx , descr_dict in enumerate ( description_dicts ) : if idx == 0 : tblname = tablename else : tblname = tablename + '_%d' % idx table = self . _hdf5file . create_table ( where = hdf5_group , name = tblname , description = descr_dict , title = tblname , expectedrows = datasize , filters = self . _all_get_filters ( kwargs . copy ( ) ) ) row = table . row for n in range ( datasize ) : # Fill the columns with data, note if the parameter was extended nstart!=0 for key in descr_dict : row [ key ] = data [ key ] [ n ] row . append ( ) # Remember the original types of the data for perfect recall if idx == 0 and len ( description_dict ) <= ptpa . MAX_COLUMNS : # We only have a single table and # we can store the original data types as attributes for field_name in data_type_dict : type_description = data_type_dict [ field_name ] self . _all_set_attr ( table , field_name , type_description ) setattr ( table . _v_attrs , HDF5StorageService . STORAGE_TYPE , HDF5StorageService . TABLE ) table . flush ( ) self . _hdf5file . flush ( ) if len ( description_dict ) > ptpa . MAX_COLUMNS : # We have potentially many split tables and the data types are # stored into an additional table for performance reasons tblname = tablename + '__' + HDF5StorageService . STORAGE_TYPE field_names , data_types = list ( zip ( * data_type_dict . items ( ) ) ) data_type_table_dict = { 'field_name' : field_names , 'data_type' : data_types } descr_dict , _ = self . _prm_make_description ( data_type_table_dict , fullname ) table = self . _hdf5file . create_table ( where = hdf5_group , name = tblname , description = descr_dict , title = tblname , expectedrows = len ( field_names ) , filters = self . _all_get_filters ( kwargs ) ) row = table . row for n in range ( len ( field_names ) ) : # Fill the columns with data for key in data_type_table_dict : row [ key ] = data_type_table_dict [ key ] [ n ] row . append ( ) setattr ( table . _v_attrs , HDF5StorageService . DATATYPE_TABLE , 1 ) table . flush ( ) self . _hdf5file . flush ( ) except : self . _logger . error ( 'Failed storing table `%s` of `%s`.' % ( tablename , fullname ) ) raise
Stores data as pytable.

        :param tablename:

            Name of the data table

        :param data:

            Data to store

        :param hdf5_group:

            Group node where to store data in hdf5 file

        :param fullname:

            Full name of the `data_to_store`s original container, only needed for throwing errors.
Stores data as pytable .