<s> """<STR_LIT>""" <EOL> import shutil , tempfile <EOL> from array import array <EOL> from bisect import bisect_left , bisect_right <EOL> from marshal import loads <EOL> from marshal import dumps <EOL> try : <EOL> from zlib import compress , decompress <EOL> has_zlib = True <EOL> except ImportError : <EOL> has_zlib = False <EOL> from whoosh . structfile import _USHORT_SIZE , StructFile <EOL> def copy_data ( treader , inkey , twriter , outkey , postings = False , buffersize = <NUM_LIT:32> * <NUM_LIT> ) : <EOL> """<STR_LIT>""" <EOL> if postings : <EOL> ( offset , length ) , postcount , data = treader . _get ( inkey ) <EOL> super ( twriter . __class__ , twriter ) . add_row ( outkey , ( ( twriter . offset , length ) , postcount , data ) ) <EOL> infile = treader . table_file <EOL> infile . seek ( treader . postpos + offset ) <EOL> outfile = twriter . posting_file <EOL> if length <= buffersize : <EOL> outfile . write ( infile . read ( length ) ) <EOL> else : <EOL> sofar = <NUM_LIT:0> <EOL> while sofar < length : <EOL> readsize = min ( buffersize , length - sofar ) <EOL> outfile . write ( infile . read ( readsize ) ) <EOL> sofar += readsize <EOL> twriter . offset = outfile . tell ( ) <EOL> else : <EOL> twriter . add_row ( outkey , treader [ inkey ] ) <EOL> class TableWriter ( object ) : <EOL> def __init__ ( self , table_file , blocksize = <NUM_LIT:16> * <NUM_LIT> , <EOL> compressed = <NUM_LIT:0> , prefixcoding = False , <EOL> postings = False , stringids = False , <EOL> checksize = True ) : <EOL> self . table_file = table_file <EOL> self . blocksize = blocksize <EOL> if compressed > <NUM_LIT:0> and not has_zlib : <EOL> raise Exception ( "<STR_LIT>" ) <EOL> self . compressed = compressed <EOL> self . prefixcoding = prefixcoding <EOL> self . haspostings = postings <EOL> if postings : <EOL> self . offset = <NUM_LIT:0> <EOL> self . postcount = <NUM_LIT:0> <EOL> self . lastpostid = None <EOL> self . stringids = stringids <EOL> self . posting_file = StructFile ( tempfile . TemporaryFile ( ) ) <EOL> self . rowbuffer = [ ] <EOL> self . lastkey = None <EOL> self . blockfilled = <NUM_LIT:0> <EOL> self . keys = [ ] <EOL> self . pointers = array ( "<STR_LIT:L>" ) <EOL> self . start = table_file . tell ( ) <EOL> table_file . write_ulong ( <NUM_LIT:0> ) <EOL> table_file . write_ulong ( <NUM_LIT:0> ) <EOL> self . options = { "<STR_LIT>" : postings , <EOL> "<STR_LIT>" : compressed , <EOL> "<STR_LIT>" : prefixcoding , <EOL> "<STR_LIT>" : stringids } <EOL> def close ( self ) : <EOL> if self . rowbuffer : <EOL> self . _write_block ( ) <EOL> tf = self . table_file <EOL> haspostings = self . haspostings <EOL> dirpos = tf . tell ( ) <EOL> tf . write_pickle ( self . keys ) <EOL> tf . write_array ( self . pointers ) <EOL> tf . write_pickle ( self . options ) <EOL> if haspostings : <EOL> postpos = tf . tell ( ) <EOL> self . posting_file . seek ( <NUM_LIT:0> ) <EOL> shutil . copyfileobj ( self . posting_file , tf ) <EOL> self . posting_file . close ( ) <EOL> tf . seek ( self . start ) <EOL> tf . write_ulong ( dirpos ) <EOL> if haspostings : <EOL> tf . write_ulong ( postpos ) <EOL> tf . close ( ) <EOL> def _write_block ( self ) : <EOL> buf = self . rowbuffer <EOL> key = buf [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] <EOL> compressed = self . compressed <EOL> self . keys . append ( key ) <EOL> self . pointers . append ( self . table_file . tell ( ) ) <EOL> if compressed : <EOL> pck = dumps ( buf ) <EOL> self . table_file . write_string ( compress ( pck , compressed ) ) <EOL> else : <EOL> self . table_file . write_pickle ( buf ) <EOL> self . rowbuffer = [ ] <EOL> self . blockfilled = <NUM_LIT:0> <EOL> def write_posting ( self , id , data , writefn ) : <EOL> if id <= self . lastpostid : <EOL> raise IndexError ( "<STR_LIT>" % ( self . lastpostid , id ) ) <EOL> pf = self . posting_file <EOL> if self . stringids : <EOL> pf . write_string ( id . encode ( "<STR_LIT:utf8>" ) ) <EOL> else : <EOL> lastpostid = self . lastpostid or <NUM_LIT:0> <EOL> pf . write_varint ( id - lastpostid ) <EOL> self . lastpostid = id <EOL> self . postcount += <NUM_LIT:1> <EOL> return writefn ( pf , data ) <EOL> def add_row ( self , key , data ) : <EOL> if key <= self . lastkey : <EOL> raise IndexError ( "<STR_LIT>" % ( self . lastkey , key ) ) <EOL> rb = self . rowbuffer <EOL> if isinstance ( data , array ) : <EOL> self . blockfilled += len ( data ) * data . itemsize <EOL> else : <EOL> self . blockfilled += len ( dumps ( data ) ) <EOL> self . lastkey = key <EOL> if self . haspostings : <EOL> endoffset = self . posting_file . tell ( ) <EOL> length = endoffset - self . offset <EOL> rb . append ( ( key , ( self . offset , length , self . postcount , data ) ) ) <EOL> self . offset = endoffset <EOL> self . postcount = <NUM_LIT:0> <EOL> self . lastpostid = None <EOL> else : <EOL> rb . append ( ( key , data ) ) <EOL> if self . blockfilled >= self . blocksize : <EOL> self . _write_block ( ) <EOL> class TableReader ( object ) : <EOL> def __init__ ( self , table_file ) : <EOL> self . table_file = table_file <EOL> dirpos = table_file . read_ulong ( ) <EOL> self . postpos = table_file . read_ulong ( ) <EOL> table_file . seek ( dirpos ) <EOL> self . blockindex = table_file . read_pickle ( ) <EOL> self . blockcount = len ( self . blockindex ) <EOL> self . blockpositions = table_file . read_array ( "<STR_LIT:L>" , self . blockcount ) <EOL> options = table_file . read_pickle ( ) <EOL> self . __dict__ . update ( options ) <EOL> if self . compressed > <NUM_LIT:0> and not has_zlib : <EOL> raise Exception ( "<STR_LIT>" ) <EOL> self . currentblock = None <EOL> self . itemlist = None <EOL> self . itemdict = None <EOL> if self . haspostings : <EOL> if self . stringids : <EOL> self . _read_id = self . _read_id_string <EOL> else : <EOL> self . _read_id = self . _read_id_varint <EOL> self . get = self . _get_ignore_postinfo <EOL> else : <EOL> self . get = self . _get_plain <EOL> def __contains__ ( self , key ) : <EOL> if key < self . blockindex [ <NUM_LIT:0> ] : <EOL> return False <EOL> self . _load_block ( key ) <EOL> return key in self . itemdict <EOL> def _get_ignore_postinfo ( self , key ) : <EOL> self . _load_block ( key ) <EOL> return self . itemdict [ key ] [ <NUM_LIT:3> ] <EOL> def _get_plain ( self , key ) : <EOL> self . _load_block ( key ) <EOL> return self . itemdict [ key ] <EOL> def __iter__ ( self ) : <EOL> if self . haspostings : <EOL> for i in xrange ( <NUM_LIT:0> , self . blockcount ) : <EOL> self . _load_block_num ( i ) <EOL> for key , value in self . itemlist : <EOL> yield ( key , value [ <NUM_LIT:3> ] ) <EOL> else : <EOL> for i in xrange ( <NUM_LIT:0> , self . blockcount ) : <EOL> self . _load_block_num ( i ) <EOL> for key , value in self . itemlist : <EOL> yield ( key , value ) <EOL> def _read_id_varint ( self , lastid ) : <EOL> return lastid + self . table_file . read_varint ( ) <EOL> def _read_id_string ( self , lastid ) : <EOL> return self . table_file . read_string ( ) . decode ( "<STR_LIT:utf8>" ) <EOL> def iter_from ( self , key ) : <EOL> postings = self . haspostings <EOL> self . _load_block ( key ) <EOL> blockcount = self . blockcount <EOL> itemlist = self . itemlist <EOL> p = bisect_left ( itemlist , ( key , None ) ) <EOL> if p >= len ( itemlist ) : <EOL> if self . currentblock >= blockcount - <NUM_LIT:1> : <EOL> return <EOL> self . _load_block_num ( self . currentblock + <NUM_LIT:1> ) <EOL> itemlist = self . itemlist <EOL> p = <NUM_LIT:0> <EOL> while True : <EOL> kv = itemlist [ p ] <EOL> if postings : <EOL> yield ( kv [ <NUM_LIT:0> ] , kv [ <NUM_LIT:1> ] [ <NUM_LIT:3> ] ) <EOL> else : <EOL> yield kv <EOL> p += <NUM_LIT:1> <EOL> if p >= len ( itemlist ) : <EOL> if self . currentblock >= blockcount - <NUM_LIT:1> : <EOL> return <EOL> self . _load_block_num ( self . currentblock + <NUM_LIT:1> ) <EOL> itemlist = self . itemlist <EOL> p = <NUM_LIT:0> <EOL> def close ( self ) : <EOL> self . table_file . close ( ) <EOL> def keys ( self ) : <EOL> return ( key for key , _ in self ) <EOL> def values ( self ) : <EOL> return ( value for _ , value in self ) <EOL> def posting_count ( self , key ) : <EOL> if not self . haspostings : raise Exception ( "<STR_LIT>" ) <EOL> return self . _get_plain ( key ) [ <NUM_LIT:2> ] <EOL> def postings ( self , key , readfn ) : <EOL> postfile = self . table_file <EOL> _read_id = self . _read_id <EOL> id = <NUM_LIT:0> <EOL> for _ in xrange ( <NUM_LIT:0> , self . _seek_postings ( key ) ) : <EOL> id = _read_id ( id ) <EOL> yield ( id , readfn ( postfile ) ) <EOL> def _load_block_num ( self , bn ) : <EOL> blockcount = len ( self . blockindex ) <EOL> if bn < <NUM_LIT:0> or bn >= blockcount : <EOL> raise ValueError ( "<STR_LIT>" % ( bn , blockcount ) ) <EOL> pos = self . blockpositions [ bn ] <EOL> self . table_file . seek ( pos ) <EOL> if self . compressed : <EOL> pck = self . table_file . read_string ( ) <EOL> itemlist = loads ( decompress ( pck ) ) <EOL> else : <EOL> itemlist = self . table_file . read_pickle ( ) <EOL> self . itemlist = itemlist <EOL> self . itemdict = dict ( itemlist ) <EOL> self . currentblock = bn <EOL> self . minkey = itemlist [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] <EOL> self . maxkey = itemlist [ - <NUM_LIT:1> ] [ <NUM_LIT:0> ] <EOL> def _load_block ( self , key ) : <EOL> if self . currentblock is None or key < self . minkey or key > self . maxkey : <EOL> bn = max ( <NUM_LIT:0> , bisect_right ( self . blockindex , key ) - <NUM_LIT:1> ) <EOL> self . _load_block_num ( bn ) <EOL> def _seek_postings ( self , key ) : <EOL> offset , length , count = self . _get_plain ( key ) [ : <NUM_LIT:3> ] <EOL> self . table_file . seek ( self . postpos + offset ) <EOL> return count <EOL> class ArrayWriter ( object ) : <EOL> def __init__ ( self , table_file , typecode , bufferlength = <NUM_LIT:4> * <NUM_LIT> ) : <EOL> if typecode not in table_file . _type_writers : <EOL> raise Exception ( "<STR_LIT>" % typecode ) <EOL> self . table_file = table_file <EOL> self . typecode = typecode <EOL> self . bufferlength = bufferlength <EOL> self . dir = { } <EOL> self . buffer = array ( typecode ) <EOL> self . start = table_file . tell ( ) <EOL> table_file . write_ulong ( <NUM_LIT:0> ) <EOL> def _flush ( self ) : <EOL> buff = self . buffer <EOL> if buff : <EOL> self . table_file . write_array ( buff ) <EOL> self . buffer = array ( self . typecode ) <EOL> def close ( self ) : <EOL> self . _flush ( ) <EOL> tf = self . table_file <EOL> dirpos = tf . tell ( ) <EOL> tf . write_pickle ( ( self . typecode , self . dir ) ) <EOL> tf . seek ( self . start ) <EOL> tf . write_ulong ( dirpos ) <EOL> tf . close ( ) <EOL> def add_row ( self , key , values = None ) : <EOL> self . _flush ( ) <EOL> self . dir [ key ] = self . table_file . tell ( ) <EOL> if values : <EOL> self . extend ( values ) <EOL> def append ( self , value ) : <EOL> buff = self . buffer <EOL> buff . append ( value ) <EOL> if len ( buff ) > self . bufferlength : <EOL> self . _flush ( ) <EOL> def extend ( self , values ) : <EOL> buff = self . buffer <EOL> buff . extend ( values ) <EOL> if len ( buff ) > self . bufferlength : <EOL> self . _flush ( ) <EOL> def from_file ( self , fobj ) : <EOL> self . _flush ( ) <EOL> shutil . copyfileobj ( fobj , self . table_file ) <EOL> class ArrayReader ( object ) : <EOL> def __init__ ( self , table_file ) : <EOL> self . table_file = table_file <EOL> dirpos = table_file . read_ulong ( ) <EOL> table_file . seek ( dirpos ) <EOL> typecode , self . dir = table_file . read_pickle ( ) <EOL> try : <EOL> self . read = self . table_file . _type_readers [ typecode ] <EOL> except KeyError : <EOL> raise Exception ( "<STR_LIT>" % self . typecode ) <EOL> self . typecode = typecode <EOL> self . itemsize = array ( typecode ) . itemsize <EOL> def __contains__ ( self , key ) : <EOL> return key in self . dir <EOL> def get ( self , key , offset ) : <EOL> tf = self . table_file <EOL> pos = self . dir [ key ] <EOL> tf . seek ( pos + offset * self . itemsize ) <EOL> return self . read ( ) <EOL> def close ( self ) : <EOL> self . table_file . close ( ) <EOL> def to_file ( self , key , fobj ) : <EOL> raise NotImplementedError <EOL> class RecordWriter ( object ) : <EOL> def __init__ ( self , table_file , typecode , length ) : <EOL> self . table_file = table_file <EOL> self . typecode = typecode <EOL> self . length = length <EOL> table_file . write ( typecode [ <NUM_LIT:0> ] ) <EOL> table_file . write_ushort ( length ) <EOL> def close ( self ) : <EOL> self . table_file . close ( ) <EOL> def append ( self , arry ) : <EOL> assert arry . typecode == self . typecode <EOL> assert len ( arry ) == self . length <EOL> self . table_file . write_array ( arry ) <EOL> class RecordReader ( object ) : <EOL> def __init__ ( self , table_file ) : <EOL> self . table_file = table_file <EOL> self . typecode = table_file . read ( <NUM_LIT:1> ) <EOL> try : <EOL> self . read = self . table_file . _type_readers [ self . typecode ] <EOL> except KeyError : <EOL> raise Exception ( "<STR_LIT>" % self . typecode ) <EOL> self . length = table_file . read_ushort ( ) <EOL> self . itemsize = array ( self . typecode ) . itemsize <EOL> self . recordsize = self . length * self . itemsize <EOL> def close ( self ) : <EOL> self . table_file . close ( ) <EOL> def get ( self , recordnum , itemnum ) : <EOL> assert itemnum < self . length <EOL> self . table_file . seek ( <NUM_LIT:1> + _USHORT_SIZE + recordnum * self . recordsize + itemnum * self . itemsize ) <EOL> return self . read ( ) <EOL> def get_record ( self , recordnum ) : <EOL> tf = self . table_file <EOL> tf . seek ( <NUM_LIT:1> + _USHORT_SIZE + recordnum * self . recordsize ) <EOL> return tf . read_array ( self . typecode , self . length ) <EOL> class StringListWriter ( object ) : <EOL> def __init__ ( self , table_file , listlength ) : <EOL> self . table_file = table_file <EOL> self . listlength = listlength <EOL> self . positions = array ( "<STR_LIT:L>" ) <EOL> table_file . write_ulong ( <NUM_LIT:0> ) <EOL> def close ( self ) : <EOL> tf = self . table_file <EOL> directory_pos = tf . tell ( ) <EOL> tf . write_array ( self . positions ) <EOL> tf . seek ( <NUM_LIT:0> ) <EOL> tf . write_ulong ( directory_pos ) <EOL> tf . close ( ) <EOL> def append ( self , ustrings ) : <EOL> assert len ( ustrings ) == self . listlength <EOL> tf = self . table_file <EOL> self . positions . append ( tf . tell ( ) ) <EOL> encoded = [ ustring . encode ( "<STR_LIT:utf8>" ) for ustring in ustrings ] <EOL> lenarray = array ( "<STR_LIT:I>" , ( len ( s ) for s in encoded ) ) <EOL> tf . write_array ( lenarray ) <EOL> tf . write ( "<STR_LIT>" . join ( encoded ) ) <EOL> class StringListReader ( object ) : <EOL> def __init__ ( self , table_file , listlength , size ) : <EOL> self . table_file = table_file <EOL> self . listlength = listlength <EOL> self . size = size <EOL> self . positions = table_file . read_array ( "<STR_LIT:L>" , size ) <EOL> def close ( self ) : <EOL> self . table_file . close ( ) <EOL> def get ( self , num ) : <EOL> tf = self . table_file <EOL> listlength = self . listlength <EOL> tf . seek ( self . positions [ num ] ) <EOL> lens = tf . read_array ( "<STR_LIT:I>" , listlength ) <EOL> string = tf . read ( sum ( lens ) ) <EOL> p = <NUM_LIT:0> <EOL> decoded = [ ] <EOL> for ln in lens : <EOL> decoded . append ( string [ p : p + ln ] . decode ( "<STR_LIT:utf8>" ) ) <EOL> p += ln <EOL> return decoded <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> pass </s>
