<s> from array import array <EOL> from struct import Struct , pack <EOL> from whoosh . compat import loads , dumps , b , bytes_type , string_type <EOL> from whoosh . matching import Matcher , ReadTooFar <EOL> from whoosh . reading import TermInfo <EOL> from whoosh . spans import Span <EOL> from whoosh . system import ( _INT_SIZE , _FLOAT_SIZE , pack_long , unpack_long , <EOL> IS_LITTLE ) <EOL> from whoosh . util import byte_to_length , length_to_byte <EOL> try : <EOL> from zlib import compress , decompress <EOL> can_compress = True <EOL> except ImportError : <EOL> can_compress = False <EOL> class Codec ( object ) : <EOL> def __init__ ( self , storage ) : <EOL> self . storage = storage <EOL> def per_document_writer ( self , segment ) : <EOL> raise NotImplementedError <EOL> def field_writer ( self , segment ) : <EOL> raise NotImplementedError <EOL> def terms_reader ( self , segment ) : <EOL> raise NotImplementedError <EOL> def lengths_reader ( self , segment ) : <EOL> raise NotImplementedError <EOL> def vector_reader ( self , segment ) : <EOL> raise NotImplementedError <EOL> def stored_fields_reader ( self , segment ) : <EOL> raise NotImplementedError <EOL> def word_graph_reader ( self , segment ) : <EOL> raise NotImplementedError <EOL> def commit_toc ( self , indexname , schema , segments , generation ) : <EOL> raise NotImplementedError <EOL> class PerDocumentWriter ( object ) : <EOL> def start_doc ( self , docnum ) : <EOL> raise NotImplementedError <EOL> def add_field ( self , fieldname , fieldobj , value , length ) : <EOL> raise NotImplementedError <EOL> def add_vector_items ( self , fieldname , fieldobj , items ) : <EOL> raise NotImplementedError <EOL> def add_vector_matcher ( self , fieldname , fieldobj , vmatcher ) : <EOL> raise NotImplementedError <EOL> def finish_doc ( self ) : <EOL> pass <EOL> def lengths_reader ( self ) : <EOL> raise NotImplementedError <EOL> class FieldWriter ( object ) : <EOL> def add_iter ( self , schema , lengths , items ) : <EOL> lastfn = None <EOL> lasttext = None <EOL> getlen = lengths . get <EOL> add = self . add <EOL> for fieldname , text , docnum , weight , valuestring in items : <EOL> if docnum is None and ( fieldname != lastfn or text != lasttext ) : <EOL> self . add_spell_word ( fieldname , text ) <EOL> lastfn = fieldname <EOL> lasttext = text <EOL> continue <EOL> if fieldname < lastfn or ( fieldname == lastfn and text < lasttext ) : <EOL> raise Exception ( "<STR_LIT>" % <EOL> ( lastfn , lasttext , fieldname , text ) ) <EOL> if fieldname != lastfn or text != lasttext : <EOL> if lasttext is not None : <EOL> self . finish_term ( ) <EOL> if fieldname != lastfn : <EOL> if lastfn is not None : <EOL> self . finish_field ( ) <EOL> self . start_field ( fieldname , schema [ fieldname ] ) <EOL> lastfn = fieldname <EOL> self . start_term ( text ) <EOL> lasttext = text <EOL> length = getlen ( docnum , fieldname ) <EOL> add ( docnum , weight , valuestring , length ) <EOL> if lasttext is not None : <EOL> self . finish_term ( ) <EOL> self . finish_field ( ) <EOL> def start_field ( self , fieldname , fieldobj ) : <EOL> raise NotImplementedError <EOL> def start_term ( self , text ) : <EOL> raise NotImplementedError <EOL> def add ( self , docnum , weight , valuestring , length ) : <EOL> raise NotImplementedError <EOL> def add_spell_word ( self , fieldname , text ) : <EOL> raise NotImplementedError <EOL> def finish_term ( self ) : <EOL> raise NotImplementedError <EOL> def finish_field ( self ) : <EOL> pass <EOL> def close ( self ) : <EOL> pass <EOL> class TermsReader ( object ) : <EOL> def __contains__ ( self , term ) : <EOL> raise NotImplementedError <EOL> def terminfo ( self , fieldname , text ) : <EOL> raise NotImplementedError <EOL> def word_graph ( self , fieldname , text ) : <EOL> raise NotImplementedError <EOL> def matcher ( self , fieldname , text , fmt ) : <EOL> raise NotImplementedError <EOL> def close ( self ) : <EOL> pass <EOL> class VectorReader ( object ) : <EOL> def __contains__ ( self , key ) : <EOL> raise NotImplementedError <EOL> def matcher ( self , docnum , fieldname , format_ ) : <EOL> raise NotImplementedError <EOL> class LengthsReader ( object ) : <EOL> def get ( self , docnum , fieldname ) : <EOL> raise NotImplementedError <EOL> def field_length ( self , fieldname ) : <EOL> raise NotImplementedError <EOL> def min_field_length ( self , fieldname ) : <EOL> raise NotImplementedError <EOL> def max_field_length ( self , fieldname ) : <EOL> raise NotImplementedError <EOL> def close ( self ) : <EOL> pass <EOL> class StoredFieldsReader ( object ) : <EOL> def __iter__ ( self ) : <EOL> raise NotImplementedError <EOL> def __getitem__ ( self , docnum ) : <EOL> raise NotImplementedError <EOL> def cell ( self , docnum , fieldname ) : <EOL> fielddict = self . get ( docnum ) <EOL> return fielddict . get ( fieldname ) <EOL> def column ( self , fieldname ) : <EOL> for fielddict in self : <EOL> yield fielddict . get ( fieldname ) <EOL> def close ( self ) : <EOL> pass <EOL> class FilePostingMatcher ( Matcher ) : <EOL> def __repr__ ( self ) : <EOL> return "<STR_LIT>" % ( self . __class__ . __name__ , str ( self . postfile ) , <EOL> self . term ( ) , self . is_active ( ) ) <EOL> def term ( self ) : <EOL> return self . _term <EOL> def items_as ( self , astype ) : <EOL> decoder = self . format . decoder ( astype ) <EOL> for id , value in self . all_items ( ) : <EOL> yield ( id , decoder ( value ) ) <EOL> def supports ( self , astype ) : <EOL> return self . format . supports ( astype ) <EOL> def value_as ( self , astype ) : <EOL> decoder = self . format . decoder ( astype ) <EOL> return decoder ( self . value ( ) ) <EOL> def spans ( self ) : <EOL> if self . supports ( "<STR_LIT>" ) : <EOL> return [ Span ( pos , startchar = startchar , endchar = endchar ) <EOL> for pos , startchar , endchar in self . value_as ( "<STR_LIT>" ) ] <EOL> elif self . supports ( "<STR_LIT>" ) : <EOL> return [ Span ( pos ) for pos in self . value_as ( "<STR_LIT>" ) ] <EOL> else : <EOL> raise Exception ( "<STR_LIT>" <EOL> % self . _term ) <EOL> def supports_block_quality ( self ) : <EOL> return self . scorer and self . scorer . supports_block_quality ( ) <EOL> def max_quality ( self ) : <EOL> return self . scorer . max_quality <EOL> def block_quality ( self ) : <EOL> return self . scorer . block_quality ( self ) <EOL> class BlockPostingMatcher ( FilePostingMatcher ) : <EOL> def id ( self ) : <EOL> return self . block . ids [ self . i ] <EOL> def weight ( self ) : <EOL> weights = self . block . weights <EOL> if weights is None : <EOL> weights = self . block . read_weights ( ) <EOL> return weights [ self . i ] <EOL> def value ( self ) : <EOL> values = self . block . values <EOL> if values is None : <EOL> values = self . block . read_values ( ) <EOL> return values [ self . i ] <EOL> def all_ids ( self ) : <EOL> nextoffset = self . baseoffset <EOL> for _ in xrange ( self . blockcount ) : <EOL> block = self . _read_block ( nextoffset ) <EOL> nextoffset = block . nextoffset <EOL> ids = block . read_ids ( ) <EOL> for id in ids : <EOL> yield id <EOL> def next ( self ) : <EOL> if self . i == self . block . count - <NUM_LIT:1> : <EOL> self . _next_block ( ) <EOL> return True <EOL> else : <EOL> self . i += <NUM_LIT:1> <EOL> return False <EOL> def skip_to ( self , id ) : <EOL> if not self . is_active ( ) : <EOL> raise ReadTooFar <EOL> i = self . i <EOL> if id <= self . block . ids [ i ] : <EOL> return <EOL> if id > self . block . maxid : <EOL> self . _skip_to_block ( lambda : id > self . block . maxid ) <EOL> if not self . is_active ( ) : <EOL> return <EOL> ids = self . block . ids <EOL> i = self . i <EOL> while ids [ i ] < id : <EOL> i += <NUM_LIT:1> <EOL> if i == len ( ids ) : <EOL> self . _active = False <EOL> return <EOL> self . i = i <EOL> def skip_to_quality ( self , minquality ) : <EOL> bq = self . block_quality <EOL> if bq ( ) > minquality : <EOL> return <NUM_LIT:0> <EOL> return self . _skip_to_block ( lambda : bq ( ) <= minquality ) <EOL> def block_min_length ( self ) : <EOL> return self . block . min_length ( ) <EOL> def block_max_length ( self ) : <EOL> return self . block . max_length ( ) <EOL> def block_max_weight ( self ) : <EOL> return self . block . max_weight ( ) <EOL> def block_max_wol ( self ) : <EOL> return self . block . max_wol ( ) <EOL> NO_ID = <NUM_LIT> <EOL> class FileTermInfo ( TermInfo ) : <EOL> struct = Struct ( "<STR_LIT>" ) <EOL> def __init__ ( self , * args , ** kwargs ) : <EOL> self . postings = None <EOL> if "<STR_LIT>" in kwargs : <EOL> self . postings = kwargs [ "<STR_LIT>" ] <EOL> del kwargs [ "<STR_LIT>" ] <EOL> TermInfo . __init__ ( self , * args , ** kwargs ) <EOL> def add_block ( self , block ) : <EOL> self . _weight += sum ( block . weights ) <EOL> self . _df += len ( block ) <EOL> ml = block . min_length ( ) <EOL> if self . _minlength is None : <EOL> self . _minlength = ml <EOL> else : <EOL> self . _minlength = min ( self . _minlength , ml ) <EOL> self . _maxlength = max ( self . _maxlength , block . max_length ( ) ) <EOL> self . _maxweight = max ( self . _maxweight , block . max_weight ( ) ) <EOL> if self . _minid is None : <EOL> self . _minid = block . ids [ <NUM_LIT:0> ] <EOL> self . _maxid = block . ids [ - <NUM_LIT:1> ] <EOL> def to_string ( self ) : <EOL> ml = <NUM_LIT:0> if self . _minlength is None else length_to_byte ( self . _minlength ) <EOL> xl = length_to_byte ( self . _maxlength ) <EOL> mid = NO_ID if self . _minid is None else self . _minid <EOL> xid = NO_ID if self . _maxid is None else self . _maxid <EOL> st = self . struct . pack ( self . _weight , self . _df , ml , xl , self . _maxweight , <EOL> <NUM_LIT:0> , mid , xid ) <EOL> if isinstance ( self . postings , tuple ) : <EOL> isinlined = <NUM_LIT:1> <EOL> st += dumps ( self . postings , - <NUM_LIT:1> ) [ <NUM_LIT:2> : - <NUM_LIT:1> ] <EOL> else : <EOL> isinlined = <NUM_LIT:0> <EOL> p = - <NUM_LIT:1> if self . postings is None else self . postings <EOL> st += pack_long ( p ) <EOL> return pack ( "<STR_LIT:B>" , isinlined ) + st <EOL> @ classmethod <EOL> def from_string ( cls , s ) : <EOL> assert isinstance ( s , bytes_type ) <EOL> if isinstance ( s , string_type ) : <EOL> hbyte = ord ( s [ <NUM_LIT:0> ] ) <EOL> else : <EOL> hbyte = s [ <NUM_LIT:0> ] <EOL> if hbyte < <NUM_LIT:2> : <EOL> st = cls . struct <EOL> w , df , ml , xl , xw , _ , mid , xid = st . unpack ( s [ <NUM_LIT:1> : st . size + <NUM_LIT:1> ] ) <EOL> mid = None if mid == NO_ID else mid <EOL> xid = None if xid == NO_ID else xid <EOL> pstr = s [ st . size + <NUM_LIT:1> : ] <EOL> if hbyte == <NUM_LIT:0> : <EOL> p = unpack_long ( pstr ) [ <NUM_LIT:0> ] <EOL> else : <EOL> p = loads ( pstr + b ( "<STR_LIT:.>" ) ) <EOL> else : <EOL> v = loads ( s + b ( "<STR_LIT:.>" ) ) <EOL> if len ( v ) == <NUM_LIT:1> : <EOL> w = df = <NUM_LIT:1> <EOL> p = v [ <NUM_LIT:0> ] <EOL> elif len ( v ) == <NUM_LIT:2> : <EOL> w = df = v [ <NUM_LIT:1> ] <EOL> p = v [ <NUM_LIT:0> ] <EOL> else : <EOL> w , p , df = v <EOL> ml = <NUM_LIT:1> <EOL> xl = <NUM_LIT:255> <EOL> xw = <NUM_LIT> <EOL> mid = - <NUM_LIT:1> <EOL> xid = - <NUM_LIT:1> <EOL> ml = byte_to_length ( ml ) <EOL> xl = byte_to_length ( xl ) <EOL> obj = cls ( w , df , ml , xl , xw , mid , xid ) <EOL> obj . postings = p <EOL> return obj <EOL> @ classmethod <EOL> def read_weight ( cls , dbfile , datapos ) : <EOL> return dbfile . get_float ( datapos + <NUM_LIT:1> ) <EOL> @ classmethod <EOL> def read_doc_freq ( cls , dbfile , datapos ) : <EOL> return dbfile . get_uint ( datapos + <NUM_LIT:1> + _FLOAT_SIZE ) <EOL> @ classmethod <EOL> def read_min_and_max_length ( cls , dbfile , datapos ) : <EOL> lenpos = datapos + <NUM_LIT:1> + _FLOAT_SIZE + _INT_SIZE <EOL> ml = byte_to_length ( dbfile . get_byte ( lenpos ) ) <EOL> xl = byte_to_length ( dbfile . get_byte ( lenpos + <NUM_LIT:1> ) ) <EOL> return ml , xl <EOL> @ classmethod <EOL> def read_max_weight ( cls , dbfile , datapos ) : <EOL> weightspos = datapos + <NUM_LIT:1> + _FLOAT_SIZE + _INT_SIZE + <NUM_LIT:2> <EOL> return dbfile . get_float ( weightspos ) <EOL> class BlockBase ( object ) : <EOL> def __init__ ( self , postingsize , stringids = False ) : <EOL> self . postingsize = postingsize <EOL> self . stringids = stringids <EOL> self . ids = [ ] if stringids else array ( "<STR_LIT:I>" ) <EOL> self . weights = array ( "<STR_LIT:f>" ) <EOL> self . values = None <EOL> self . minlength = None <EOL> self . maxlength = <NUM_LIT:0> <EOL> self . maxweight = <NUM_LIT:0> <EOL> def __len__ ( self ) : <EOL> return len ( self . ids ) <EOL> def __nonzero__ ( self ) : <EOL> return bool ( self . ids ) <EOL> def min_id ( self ) : <EOL> if self . ids : <EOL> return self . ids [ <NUM_LIT:0> ] <EOL> else : <EOL> raise IndexError <EOL> def max_id ( self ) : <EOL> if self . ids : <EOL> return self . ids [ - <NUM_LIT:1> ] <EOL> else : <EOL> raise IndexError <EOL> def min_length ( self ) : <EOL> return self . minlength <EOL> def max_length ( self ) : <EOL> return self . maxlength <EOL> def max_weight ( self ) : <EOL> return self . maxweight <EOL> def add ( self , id_ , weight , valuestring , length = None ) : <EOL> self . ids . append ( id_ ) <EOL> self . weights . append ( weight ) <EOL> if weight > self . maxweight : <EOL> self . maxweight = weight <EOL> if valuestring : <EOL> if self . values is None : <EOL> self . values = [ ] <EOL> self . values . append ( valuestring ) <EOL> if length : <EOL> if self . minlength is None or length < self . minlength : <EOL> self . minlength = length <EOL> if length > self . maxlength : <EOL> self . maxlength = length <EOL> def to_file ( self , postfile ) : <EOL> raise NotImplementedError <EOL> @ classmethod <EOL> def from_file ( cls , postfile ) : <EOL> raise NotImplementedError <EOL> def minimize_ids ( arry , stringids , compression = <NUM_LIT:0> ) : <EOL> amax = arry [ - <NUM_LIT:1> ] <EOL> if stringids : <EOL> typecode = '<STR_LIT>' <EOL> string = dumps ( arry ) <EOL> else : <EOL> code = arry . typecode <EOL> if amax <= <NUM_LIT:255> : <EOL> typecode = "<STR_LIT:B>" <EOL> elif amax <= <NUM_LIT> : <EOL> typecode = "<STR_LIT:H>" <EOL> if typecode != code : <EOL> arry = array ( typecode , iter ( arry ) ) <EOL> if not IS_LITTLE : <EOL> arry . byteswap ( ) <EOL> string = arry . tostring ( ) <EOL> if compression : <EOL> string = compress ( string , compression ) <EOL> return ( typecode , string ) <EOL> def deminimize_ids ( typecode , count , string , compression = <NUM_LIT:0> ) : <EOL> if compression : <EOL> string = decompress ( string ) <EOL> if typecode == '<STR_LIT>' : <EOL> return loads ( string ) <EOL> else : <EOL> arry = array ( typecode ) <EOL> arry . fromstring ( string ) <EOL> if not IS_LITTLE : <EOL> arry . byteswap ( ) <EOL> return arry <EOL> def minimize_weights ( weights , compression = <NUM_LIT:0> ) : <EOL> if all ( w == <NUM_LIT:1.0> for w in weights ) : <EOL> string = "<STR_LIT>" <EOL> else : <EOL> if not IS_LITTLE : <EOL> weights . byteswap ( ) <EOL> string = weights . tostring ( ) <EOL> if string and compression : <EOL> string = compress ( string , compression ) <EOL> return string <EOL> def deminimize_weights ( count , string , compression = <NUM_LIT:0> ) : <EOL> if not string : <EOL> return array ( "<STR_LIT:f>" , ( <NUM_LIT:1.0> for _ in xrange ( count ) ) ) <EOL> if compression : <EOL> string = decompress ( string ) <EOL> arry = array ( "<STR_LIT:f>" ) <EOL> arry . fromstring ( string ) <EOL> if not IS_LITTLE : <EOL> arry . byteswap ( ) <EOL> return arry <EOL> def minimize_values ( postingsize , values , compression = <NUM_LIT:0> ) : <EOL> if postingsize < <NUM_LIT:0> : <EOL> string = dumps ( values , - <NUM_LIT:1> ) [ <NUM_LIT:2> : ] <EOL> elif postingsize == <NUM_LIT:0> : <EOL> string = b ( '<STR_LIT>' ) <EOL> else : <EOL> string = b ( '<STR_LIT>' ) . join ( values ) <EOL> if string and compression : <EOL> string = compress ( string , compression ) <EOL> return string <EOL> def deminimize_values ( postingsize , count , string , compression = <NUM_LIT:0> ) : <EOL> if compression : <EOL> string = decompress ( string ) <EOL> if postingsize < <NUM_LIT:0> : <EOL> return loads ( string ) <EOL> elif postingsize == <NUM_LIT:0> : <EOL> return [ None ] * count <EOL> else : <EOL> return [ string [ i : i + postingsize ] for i <EOL> in xrange ( <NUM_LIT:0> , len ( string ) , postingsize ) ] </s>
