<s> from threading import Lock <EOL> from whoosh . fields import FieldConfigurationError <EOL> from whoosh . filedb . filepostings import FilePostingReader <EOL> from whoosh . filedb . filetables import FileTableReader , FileRecordReader , FileListReader <EOL> from whoosh . filedb . filetables import encode_termkey , decode_termkey <EOL> from whoosh . filedb . filetables import encode_vectorkey , decode_vectorkey , decode_terminfo <EOL> from whoosh . filedb . filetables import depickle , unpackint <EOL> from whoosh . postings import Exclude <EOL> from whoosh . reading import IndexReader , TermNotFound <EOL> from whoosh . util import protected <EOL> def open_terms ( storage , segment ) : <EOL> termfile = storage . open_file ( segment . term_filename ) <EOL> return FileTableReader ( termfile , <EOL> keycoder = encode_termkey , <EOL> keydecoder = decode_termkey , <EOL> valuedecoder = decode_terminfo ) <EOL> def open_doclengths ( storage , segment , fieldcount ) : <EOL> from whoosh . filedb . filewriting import DOCLENGTH_TYPE <EOL> rformat = "<STR_LIT:!>" + DOCLENGTH_TYPE * fieldcount <EOL> recordfile = storage . open_file ( segment . doclen_filename ) <EOL> return FileRecordReader ( recordfile , rformat ) <EOL> def open_storedfields ( storage , segment , storedfieldnames ) : <EOL> def dictifier ( value ) : <EOL> value = depickle ( value ) <EOL> return dict ( zip ( storedfieldnames , value ) ) <EOL> listfile = storage . open_file ( segment . docs_filename , mapped = False ) <EOL> return FileListReader ( listfile , segment . doc_count_all ( ) , <EOL> valuedecoder = dictifier ) <EOL> def open_vectors ( storage , segment ) : <EOL> vectorfile = storage . open_file ( segment . vector_filename ) <EOL> return FileTableReader ( vectorfile , keycoder = encode_vectorkey , <EOL> keydecoder = decode_vectorkey , <EOL> valuedecoder = unpackint ) <EOL> class SegmentReader ( IndexReader ) : <EOL> def __init__ ( self , storage , segment , schema ) : <EOL> self . storage = storage <EOL> self . segment = segment <EOL> self . schema = schema <EOL> self . _scorable_fields = schema . scorable_fields ( ) <EOL> self . _fieldnum_to_scorable_pos = dict ( ( fnum , i ) for i , fnum <EOL> in enumerate ( self . _scorable_fields ) ) <EOL> self . termtable = open_terms ( storage , segment ) <EOL> self . postfile = None <EOL> self . docstable = open_storedfields ( storage , segment , schema . stored_field_names ( ) ) <EOL> self . doclengths = None <EOL> if self . _scorable_fields : <EOL> self . doclengths = open_doclengths ( storage , segment , len ( self . _scorable_fields ) ) <EOL> self . has_deletions = segment . has_deletions <EOL> self . is_deleted = segment . is_deleted <EOL> self . doc_count = segment . doc_count <EOL> self . doc_count_all = segment . doc_count_all <EOL> self . vectortable = None <EOL> self . is_closed = False <EOL> self . _sync_lock = Lock ( ) <EOL> def __repr__ ( self ) : <EOL> return "<STR_LIT>" % ( self . __class__ . __name__ , self . segment ) <EOL> @ protected <EOL> def __contains__ ( self , term ) : <EOL> return ( self . schema . to_number ( term [ <NUM_LIT:0> ] ) , term [ <NUM_LIT:1> ] ) in self . termtable <EOL> def close ( self ) : <EOL> self . docstable . close ( ) <EOL> self . termtable . close ( ) <EOL> if self . postfile : <EOL> self . postfile . close ( ) <EOL> if self . vectortable : <EOL> self . vectortable . close ( ) <EOL> if self . doclengths : <EOL> self . doclengths . close ( ) <EOL> self . is_closed = True <EOL> def _open_vectors ( self ) : <EOL> if not self . vectortable : <EOL> storage , segment = self . storage , self . segment <EOL> self . vectortable = open_vectors ( storage , segment ) <EOL> self . vpostfile = storage . open_file ( segment . vectorposts_filename , mapped = False ) <EOL> def vector ( self , docnum , fieldid ) : <EOL> self . _open_vectors ( ) <EOL> schema = self . schema <EOL> fieldnum = schema . to_number ( fieldid ) <EOL> vformat = schema [ fieldnum ] . vector <EOL> offset = self . vectortable [ ( docnum , fieldnum ) ] <EOL> return FilePostingReader ( self . vpostfile , offset , vformat , stringids = True ) <EOL> @ protected <EOL> def stored_fields ( self , docnum ) : <EOL> return self . docstable [ docnum ] <EOL> @ protected <EOL> def all_stored_fields ( self ) : <EOL> is_deleted = self . segment . is_deleted <EOL> for docnum in xrange ( <NUM_LIT:0> , self . segment . doc_count_all ( ) ) : <EOL> if not is_deleted ( docnum ) : <EOL> yield self . docstable [ docnum ] <EOL> def field_length ( self , fieldid ) : <EOL> fieldid = self . schema . to_number ( fieldid ) <EOL> return self . segment . field_length ( fieldid ) <EOL> @ protected <EOL> def doc_field_length ( self , docnum , fieldid ) : <EOL> fieldid = self . schema . to_number ( fieldid ) <EOL> if fieldid not in self . _scorable_fields : <EOL> raise FieldConfigurationError ( "<STR_LIT>" % fieldid ) <EOL> pos = self . _fieldnum_to_scorable_pos [ fieldid ] <EOL> return self . doclengths . at ( docnum , pos ) <EOL> @ protected <EOL> def doc_field_lengths ( self , docnum ) : <EOL> if not self . doclengths : <EOL> return [ ] <EOL> return self . doclengths . record ( docnum ) <EOL> @ protected <EOL> def has_vector ( self , docnum , fieldnum ) : <EOL> self . _open_vectors ( ) <EOL> return ( docnum , fieldnum ) in self . vectortable <EOL> @ protected <EOL> def __iter__ ( self ) : <EOL> for ( fn , t ) , ( totalfreq , offset , postcount ) in self . termtable : <EOL> yield ( fn , t , postcount , totalfreq ) <EOL> @ protected <EOL> def iter_from ( self , fieldnum , text ) : <EOL> tt = self . termtable <EOL> for ( fn , t ) , ( totalfreq , offset , postcount ) in tt . items_from ( ( fieldnum , text ) ) : <EOL> yield ( fn , t , postcount , totalfreq ) <EOL> @ protected <EOL> def _term_info ( self , fieldnum , text ) : <EOL> try : <EOL> return self . termtable [ ( fieldnum , text ) ] <EOL> except KeyError : <EOL> raise TermNotFound ( "<STR_LIT>" % ( fieldnum , text ) ) <EOL> def doc_frequency ( self , fieldid , text ) : <EOL> try : <EOL> fieldid = self . schema . to_number ( fieldid ) <EOL> return self . _term_info ( fieldid , text ) [ <NUM_LIT:2> ] <EOL> except TermNotFound : <EOL> return <NUM_LIT:0> <EOL> def frequency ( self , fieldid , text ) : <EOL> try : <EOL> fieldid = self . schema . to_number ( fieldid ) <EOL> return self . _term_info ( fieldid , text ) [ <NUM_LIT:0> ] <EOL> except TermNotFound : <EOL> return <NUM_LIT:0> <EOL> @ protected <EOL> def lexicon ( self , fieldid ) : <EOL> tt = self . termtable <EOL> fieldid = self . schema . to_number ( fieldid ) <EOL> for fn , t in tt . keys_from ( ( fieldid , '<STR_LIT>' ) ) : <EOL> if fn != fieldid : <EOL> return <EOL> yield t <EOL> @ protected <EOL> def expand_prefix ( self , fieldid , prefix ) : <EOL> tt = self . termtable <EOL> fieldid = self . schema . to_number ( fieldid ) <EOL> for fn , t in tt . keys_from ( ( fieldid , prefix ) ) : <EOL> if fn != fieldid or not t . startswith ( prefix ) : <EOL> return <EOL> yield t <EOL> def postings ( self , fieldid , text , exclude_docs = frozenset ( ) ) : <EOL> schema = self . schema <EOL> fieldnum = schema . to_number ( fieldid ) <EOL> format = schema [ fieldnum ] . format <EOL> try : <EOL> totalfreq , offset , postcount = self . termtable [ ( fieldnum , text ) ] <EOL> except KeyError : <EOL> raise TermNotFound ( "<STR_LIT>" % ( fieldid , text ) ) <EOL> if self . segment . deleted and exclude_docs : <EOL> exclude_docs = self . segment . deleted | exclude_docs <EOL> elif self . segment . deleted : <EOL> exclude_docs = self . segment . deleted <EOL> if not self . postfile : <EOL> self . postfile = self . storage . open_file ( self . segment . posts_filename , mapped = False ) <EOL> postreader = FilePostingReader ( self . postfile , offset , format ) <EOL> if exclude_docs : <EOL> postreader = Exclude ( postreader , exclude_docs ) <EOL> return postreader </s>
