<s> import mmap , os <EOL> from cPickle import dump as dump_pickle <EOL> from cPickle import load as load_pickle <EOL> from struct import calcsize , pack , unpack , Struct <EOL> from whoosh . system import _INT_SIZE , _USHORT_SIZE , _ULONG_SIZE , _FLOAT_SIZE <EOL> from whoosh . util import varint , read_varint , float_to_byte , byte_to_float <EOL> _SIZEMAP = dict ( ( typecode , calcsize ( typecode ) ) for typecode in "<STR_LIT>" ) <EOL> _ORDERMAP = { "<STR_LIT>" : "<STR_LIT:<>" , "<STR_LIT>" : "<STR_LIT:>>" } <EOL> _types = ( ( "<STR_LIT>" , "<STR_LIT:b>" ) , ( "<STR_LIT>" , "<STR_LIT:H>" ) , ( "<STR_LIT:int>" , "<STR_LIT:i>" ) , <EOL> ( "<STR_LIT>" , "<STR_LIT:L>" ) , ( "<STR_LIT:float>" , "<STR_LIT:f>" ) ) <EOL> _sbyte_struct = Struct ( "<STR_LIT>" ) <EOL> _ushort_struct = Struct ( "<STR_LIT>" ) <EOL> _int_struct = Struct ( "<STR_LIT>" ) <EOL> _uint_struct = Struct ( "<STR_LIT>" ) <EOL> _ulong_struct = Struct ( "<STR_LIT>" ) <EOL> _float_struct = Struct ( "<STR_LIT>" ) <EOL> pack_sbyte = _sbyte_struct . pack <EOL> pack_ushort = _ushort_struct . pack <EOL> pack_int = _int_struct . pack <EOL> pack_uint = _uint_struct . pack <EOL> pack_ulong = _ulong_struct . pack <EOL> pack_float = _float_struct . pack <EOL> unpack_sbyte = _sbyte_struct . unpack <EOL> unpack_ushort = _ushort_struct . unpack <EOL> unpack_int = _int_struct . unpack <EOL> unpack_uint = _uint_struct . unpack <EOL> unpack_ulong = _ulong_struct . unpack <EOL> unpack_float = _float_struct . unpack <EOL> class StructFile ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , fileobj , name = None , onclose = None , mapped = True ) : <EOL> self . file = fileobj <EOL> self . _name = name <EOL> self . onclose = onclose <EOL> self . is_closed = False <EOL> for attr in ( "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ) : <EOL> if hasattr ( fileobj , attr ) : <EOL> setattr ( self , attr , getattr ( fileobj , attr ) ) <EOL> if mapped and hasattr ( fileobj , "<STR_LIT>" ) and "<STR_LIT:r>" in fileobj . mode : <EOL> fd = fileobj . fileno ( ) <EOL> self . size = os . fstat ( fd ) . st_size <EOL> try : <EOL> self . map = mmap . mmap ( fd , self . size , access = mmap . ACCESS_READ ) <EOL> except OSError : <EOL> self . _setup_fake_map ( ) <EOL> else : <EOL> self . _setup_fake_map ( ) <EOL> def __repr__ ( self ) : <EOL> return "<STR_LIT>" % ( self . __class__ . __name__ , self . _name ) <EOL> def flush ( self ) : <EOL> """<STR_LIT>""" <EOL> if hasattr ( self . file , "<STR_LIT>" ) : <EOL> self . file . flush ( ) <EOL> def close ( self ) : <EOL> """<STR_LIT>""" <EOL> del self . map <EOL> if self . onclose : <EOL> self . onclose ( self ) <EOL> if hasattr ( self . file , "<STR_LIT>" ) : <EOL> self . file . close ( ) <EOL> self . is_closed = True <EOL> def _setup_fake_map ( self ) : <EOL> _self = self <EOL> class fakemap ( object ) : <EOL> def __getitem__ ( self , slice ) : <EOL> if isinstance ( slice , ( int , long ) ) : <EOL> _self . seek ( slice ) <EOL> return _self . read ( <NUM_LIT:1> ) <EOL> else : <EOL> _self . seek ( slice . start ) <EOL> return _self . read ( slice . stop - slice . start ) <EOL> self . map = fakemap ( ) <EOL> def write_string ( self , s ) : <EOL> """<STR_LIT>""" <EOL> self . write_varint ( len ( s ) ) <EOL> self . file . write ( s ) <EOL> def write_string2 ( self , s ) : <EOL> self . write ( pack_ushort ( len ( s ) ) + s ) <EOL> def read_string ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . file . read ( self . read_varint ( ) ) <EOL> def read_string2 ( self ) : <EOL> l = self . read_ushort ( ) <EOL> return self . read ( l ) <EOL> def skip_string ( self ) : <EOL> l = self . read_varint ( ) <EOL> self . seek ( l , <NUM_LIT:1> ) <EOL> def write_varint ( self , i ) : <EOL> """<STR_LIT>""" <EOL> self . file . write ( varint ( i ) ) <EOL> def read_varint ( self ) : <EOL> """<STR_LIT>""" <EOL> return read_varint ( self . file . read ) <EOL> def write_byte ( self , n ) : <EOL> """<STR_LIT>""" <EOL> self . file . write ( chr ( n ) ) <EOL> def read_byte ( self ) : <EOL> return ord ( self . file . read ( <NUM_LIT:1> ) ) <EOL> def get_byte ( self , position ) : <EOL> return ord ( self . map [ position ] ) <EOL> def write_8bitfloat ( self , f , mantissabits = <NUM_LIT:5> , zeroexp = <NUM_LIT:2> ) : <EOL> """<STR_LIT>""" <EOL> self . write_byte ( float_to_byte ( f , mantissabits , zeroexp ) ) <EOL> def read_8bitfloat ( self , mantissabits = <NUM_LIT:5> , zeroexp = <NUM_LIT:2> ) : <EOL> """<STR_LIT>""" <EOL> return byte_to_float ( self . read_byte ( ) , mantissabits , zeroexp ) <EOL> def write_pickle ( self , obj , protocol = - <NUM_LIT:1> ) : <EOL> """<STR_LIT>""" <EOL> dump_pickle ( obj , self . file , protocol ) <EOL> def read_pickle ( self ) : <EOL> """<STR_LIT>""" <EOL> return load_pickle ( self . file ) <EOL> def write_sbyte ( self , n ) : <EOL> self . file . write ( pack_sbyte ( n ) ) <EOL> def write_int ( self , n ) : <EOL> self . file . write ( pack_int ( n ) ) <EOL> def write_uint ( self , n ) : <EOL> self . file . write ( pack_uint ( n ) ) <EOL> def write_ushort ( self , n ) : <EOL> self . file . write ( pack_ushort ( n ) ) <EOL> def write_ulong ( self , n ) : <EOL> self . file . write ( pack_ulong ( n ) ) <EOL> def write_float ( self , n ) : <EOL> self . file . write ( pack_float ( n ) ) <EOL> def write_array ( self , arry ) : <EOL> a = Struct ( "<STR_LIT:!>" + arry . typecode * len ( arry ) ) . pack ( * arry ) <EOL> self . file . write ( a ) <EOL> def read_sbyte ( self ) : <EOL> return unpack_sbyte ( self . file . read ( <NUM_LIT:1> ) ) [ <NUM_LIT:0> ] <EOL> def read_int ( self ) : <EOL> return unpack_int ( self . file . read ( _INT_SIZE ) ) [ <NUM_LIT:0> ] <EOL> def read_uint ( self ) : <EOL> return unpack_uint ( self . file . read ( _INT_SIZE ) ) [ <NUM_LIT:0> ] <EOL> def read_ushort ( self ) : <EOL> return unpack_ushort ( self . file . read ( _USHORT_SIZE ) ) [ <NUM_LIT:0> ] <EOL> def read_ulong ( self ) : <EOL> return unpack_ulong ( self . file . read ( _ULONG_SIZE ) ) [ <NUM_LIT:0> ] <EOL> def read_float ( self ) : <EOL> return unpack_float ( self . file . read ( _FLOAT_SIZE ) ) [ <NUM_LIT:0> ] <EOL> def read_array ( self , typecode , length ) : <EOL> packed = self . file . read ( _SIZEMAP [ typecode ] * length ) <EOL> return Struct ( "<STR_LIT:!>" + typecode * length ) . unpack ( packed ) <EOL> def get_sbyte ( self , position ) : <EOL> return unpack_sbyte ( self . map [ position : position + <NUM_LIT:1> ] ) [ <NUM_LIT:0> ] <EOL> def get_int ( self , position ) : <EOL> return unpack_int ( self . map [ position : position + _INT_SIZE ] ) [ <NUM_LIT:0> ] <EOL> def get_uint ( self , position ) : <EOL> return unpack_uint ( self . map [ position : position + _INT_SIZE ] ) [ <NUM_LIT:0> ] <EOL> def get_ushort ( self , position ) : <EOL> return unpack_ushort ( self . map [ position : position + _USHORT_SIZE ] ) [ <NUM_LIT:0> ] <EOL> def get_ulong ( self , position ) : <EOL> return unpack_ulong ( self . map [ position : position + _ULONG_SIZE ] ) [ <NUM_LIT:0> ] <EOL> def get_float ( self , position ) : <EOL> return unpack_float ( self . map [ position : position + _FLOAT_SIZE ] ) [ <NUM_LIT:0> ] <EOL> def get_array ( self , position , typecode , length ) : <EOL> return unpack ( "<STR_LIT:!>" + typecode * length , <EOL> self . map [ position : position + _SIZEMAP [ typecode ] * length ] ) </s>
