<s> import random , re , sys <EOL> from time import time , sleep <EOL> from whoosh import __version__ <EOL> from whoosh . compat import pickle , integer_types , string_type , xrange <EOL> from whoosh . fields import ensure_schema <EOL> from whoosh . index import ( Index , EmptyIndexError , IndexVersionError , <EOL> _DEF_INDEX_NAME ) <EOL> from whoosh . reading import EmptyReader , MultiReader <EOL> from whoosh . store import Storage <EOL> from whoosh . system import _INT_SIZE , _FLOAT_SIZE , _LONG_SIZE <EOL> _INDEX_VERSION = - <NUM_LIT> <EOL> class TOC ( object ) : <EOL> """<STR_LIT>""" <EOL> @ classmethod <EOL> def _filename ( cls , indexname , gen ) : <EOL> return "<STR_LIT>" % ( indexname , gen ) <EOL> @ classmethod <EOL> def _pattern ( cls , indexname ) : <EOL> return re . compile ( "<STR_LIT>" % indexname ) <EOL> @ classmethod <EOL> def _segment_pattern ( cls , indexname ) : <EOL> return re . compile ( "<STR_LIT>" % indexname ) <EOL> @ classmethod <EOL> def _latest_generation ( cls , storage , indexname ) : <EOL> pattern = cls . _pattern ( indexname ) <EOL> mx = - <NUM_LIT:1> <EOL> for filename in storage : <EOL> m = pattern . match ( filename ) <EOL> if m : <EOL> mx = max ( int ( m . group ( <NUM_LIT:1> ) ) , mx ) <EOL> return mx <EOL> @ classmethod <EOL> def create ( cls , storage , schema , indexname = _DEF_INDEX_NAME ) : <EOL> schema = ensure_schema ( schema ) <EOL> prefix = "<STR_LIT>" % indexname <EOL> for filename in storage : <EOL> if filename . startswith ( prefix ) : <EOL> storage . delete_file ( filename ) <EOL> toc = cls ( schema , [ ] , <NUM_LIT:0> ) <EOL> toc . write ( storage , indexname ) <EOL> @ classmethod <EOL> def read ( cls , storage , indexname , gen = None , schema = None ) : <EOL> if gen is None : <EOL> gen = cls . _latest_generation ( storage , indexname ) <EOL> if gen < <NUM_LIT:0> : <EOL> raise EmptyIndexError ( "<STR_LIT>" <EOL> % ( indexname , storage ) ) <EOL> tocfilename = cls . _filename ( indexname , gen ) <EOL> stream = storage . open_file ( tocfilename ) <EOL> def check_size ( name , target ) : <EOL> sz = stream . read_varint ( ) <EOL> if sz != target : <EOL> raise IndexError ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> % ( name , sz , target ) ) <EOL> check_size ( "<STR_LIT:int>" , _INT_SIZE ) <EOL> check_size ( "<STR_LIT>" , _LONG_SIZE ) <EOL> check_size ( "<STR_LIT:float>" , _FLOAT_SIZE ) <EOL> if not stream . read_int ( ) == - <NUM_LIT> : <EOL> raise IndexError ( "<STR_LIT>" ) <EOL> version = stream . read_int ( ) <EOL> if version != _INDEX_VERSION : <EOL> raise IndexVersionError ( "<STR_LIT>" % version , version ) <EOL> release = ( stream . read_varint ( ) , stream . read_varint ( ) , <EOL> stream . read_varint ( ) ) <EOL> if schema : <EOL> stream . skip_string ( ) <EOL> else : <EOL> schema = pickle . loads ( stream . read_string ( ) ) <EOL> schema = ensure_schema ( schema ) <EOL> index_gen = stream . read_int ( ) <EOL> assert gen == index_gen <EOL> _ = stream . read_int ( ) <EOL> segments = stream . read_pickle ( ) <EOL> stream . close ( ) <EOL> return cls ( schema , segments , gen , version = version , release = release ) <EOL> def __init__ ( self , schema , segments , generation , <EOL> version = _INDEX_VERSION , release = __version__ ) : <EOL> self . schema = schema <EOL> self . segments = segments <EOL> self . generation = generation <EOL> self . version = version <EOL> self . release = release <EOL> def write ( self , storage , indexname ) : <EOL> schema = ensure_schema ( self . schema ) <EOL> schema . clean ( ) <EOL> tocfilename = self . _filename ( indexname , self . generation ) <EOL> tempfilename = '<STR_LIT>' % ( tocfilename , time ( ) ) <EOL> stream = storage . create_file ( tempfilename ) <EOL> stream . write_varint ( _INT_SIZE ) <EOL> stream . write_varint ( _LONG_SIZE ) <EOL> stream . write_varint ( _FLOAT_SIZE ) <EOL> stream . write_int ( - <NUM_LIT> ) <EOL> stream . write_int ( _INDEX_VERSION ) <EOL> for num in __version__ [ : <NUM_LIT:3> ] : <EOL> stream . write_varint ( num ) <EOL> stream . write_string ( pickle . dumps ( schema , - <NUM_LIT:1> ) ) <EOL> stream . write_int ( self . generation ) <EOL> stream . write_int ( <NUM_LIT:0> ) <EOL> stream . write_pickle ( self . segments ) <EOL> stream . close ( ) <EOL> storage . rename_file ( tempfilename , tocfilename , safe = True ) <EOL> def clean_files ( storage , indexname , gen , segments ) : <EOL> current_segment_names = set ( s . segment_id ( ) for s in segments ) <EOL> tocpattern = TOC . _pattern ( indexname ) <EOL> segpattern = TOC . _segment_pattern ( indexname ) <EOL> todelete = set ( ) <EOL> for filename in storage : <EOL> tocm = tocpattern . match ( filename ) <EOL> segm = segpattern . match ( filename ) <EOL> if tocm : <EOL> if int ( tocm . group ( <NUM_LIT:1> ) ) != gen : <EOL> todelete . add ( filename ) <EOL> elif segm : <EOL> name = segm . group ( <NUM_LIT:1> ) <EOL> if name not in current_segment_names : <EOL> todelete . add ( filename ) <EOL> for filename in todelete : <EOL> try : <EOL> storage . delete_file ( filename ) <EOL> except OSError : <EOL> pass <EOL> class FileIndex ( Index ) : <EOL> def __init__ ( self , storage , schema = None , indexname = _DEF_INDEX_NAME ) : <EOL> if not isinstance ( storage , Storage ) : <EOL> raise ValueError ( "<STR_LIT>" % storage ) <EOL> if not isinstance ( indexname , string_type ) : <EOL> raise ValueError ( "<STR_LIT>" % indexname ) <EOL> if schema : <EOL> schema = ensure_schema ( schema ) <EOL> self . storage = storage <EOL> self . _schema = schema <EOL> self . indexname = indexname <EOL> TOC . read ( self . storage , self . indexname , schema = self . _schema ) <EOL> def __repr__ ( self ) : <EOL> return "<STR_LIT>" % ( self . __class__ . __name__ , <EOL> self . storage , self . indexname ) <EOL> def close ( self ) : <EOL> pass <EOL> def latest_generation ( self ) : <EOL> return TOC . _latest_generation ( self . storage , self . indexname ) <EOL> def last_modified ( self ) : <EOL> gen = self . latest_generation ( ) <EOL> filename = TOC . _filename ( self . indexname , gen ) <EOL> return self . storage . file_modified ( filename ) <EOL> def is_empty ( self ) : <EOL> return len ( self . _read_toc ( ) . segments ) == <NUM_LIT:0> <EOL> def optimize ( self ) : <EOL> w = self . writer ( ) <EOL> w . commit ( optimize = True ) <EOL> def writer ( self , procs = <NUM_LIT:1> , ** kwargs ) : <EOL> if procs > <NUM_LIT:1> : <EOL> from whoosh . filedb . multiproc2 import MpWriter <EOL> return MpWriter ( self , ** kwargs ) <EOL> else : <EOL> from whoosh . filedb . filewriting import SegmentWriter <EOL> return SegmentWriter ( self , ** kwargs ) <EOL> def lock ( self , name ) : <EOL> """<STR_LIT>""" <EOL> return self . storage . lock ( self . indexname + "<STR_LIT:_>" + name ) <EOL> def _read_toc ( self ) : <EOL> return TOC . read ( self . storage , self . indexname , schema = self . _schema ) <EOL> def _segments ( self ) : <EOL> return self . _read_toc ( ) . segments <EOL> def _current_schema ( self ) : <EOL> return self . _read_toc ( ) . schema <EOL> @ property <EOL> def schema ( self ) : <EOL> return self . _current_schema ( ) <EOL> @ classmethod <EOL> def _reader ( self , storage , schema , segments , generation , reuse = None ) : <EOL> from whoosh . filedb . filereading import SegmentReader <EOL> reusable = { } <EOL> try : <EOL> if len ( segments ) == <NUM_LIT:0> : <EOL> return EmptyReader ( schema ) <EOL> if reuse : <EOL> readers = [ r for r , _ in reuse . leaf_readers ( ) ] <EOL> reusable = dict ( ( r . generation ( ) , r ) for r in readers ) <EOL> def segreader ( segment ) : <EOL> segid = segment . segment_id ( ) <EOL> if segid in reusable : <EOL> r = reusable [ segid ] <EOL> del reusable [ segid ] <EOL> return r <EOL> else : <EOL> return SegmentReader ( storage , schema , segment , <EOL> generation = generation ) <EOL> if len ( segments ) == <NUM_LIT:1> : <EOL> return segreader ( segments [ <NUM_LIT:0> ] ) <EOL> else : <EOL> readers = [ segreader ( segment ) for segment in segments ] <EOL> return MultiReader ( readers , generation = generation ) <EOL> finally : <EOL> for r in reusable . values ( ) : <EOL> r . close ( ) <EOL> def reader ( self , reuse = None ) : <EOL> retries = <NUM_LIT:10> <EOL> while retries > <NUM_LIT:0> : <EOL> try : <EOL> info = self . _read_toc ( ) <EOL> return self . _reader ( self . storage , info . schema , info . segments , <EOL> info . generation , reuse = reuse ) <EOL> except IOError : <EOL> e = sys . exc_info ( ) [ <NUM_LIT:1> ] <EOL> retries -= <NUM_LIT:1> <EOL> if retries <= <NUM_LIT:0> : <EOL> raise e <EOL> sleep ( <NUM_LIT> ) <EOL> class Segment ( object ) : <EOL> """<STR_LIT>""" <EOL> IDCHARS = "<STR_LIT>" <EOL> @ classmethod <EOL> def _random_id ( cls , size = <NUM_LIT:12> ) : <EOL> return "<STR_LIT>" . join ( random . choice ( cls . IDCHARS ) for _ in xrange ( size ) ) <EOL> def __init__ ( self , indexname , doccount = <NUM_LIT:0> , segid = None , deleted = None ) : <EOL> """<STR_LIT>""" <EOL> assert isinstance ( indexname , string_type ) <EOL> self . indexname = indexname <EOL> assert isinstance ( doccount , integer_types ) <EOL> self . doccount = doccount <EOL> self . segid = self . _random_id ( ) if segid is None else segid <EOL> self . deleted = deleted <EOL> def __repr__ ( self ) : <EOL> return "<STR_LIT>" % ( self . __class__ . __name__ , getattr ( self , "<STR_LIT>" , "<STR_LIT>" ) ) <EOL> def segment_id ( self ) : <EOL> if hasattr ( self , "<STR_LIT:name>" ) : <EOL> return self . name <EOL> else : <EOL> return "<STR_LIT>" % ( self . indexname , self . segid ) <EOL> def make_filename ( self , ext ) : <EOL> return "<STR_LIT>" % ( self . segment_id ( ) , ext ) <EOL> def create_file ( self , storage , ext , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> fname = self . make_filename ( ext ) <EOL> return storage . create_file ( fname , ** kwargs ) <EOL> def open_file ( self , storage , ext , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> fname = self . make_filename ( ext ) <EOL> return storage . open_file ( fname , ** kwargs ) <EOL> def doc_count_all ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . doccount <EOL> def doc_count ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . doccount - self . deleted_count ( ) <EOL> def has_deletions ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . deleted_count ( ) > <NUM_LIT:0> <EOL> def deleted_count ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . deleted is None : <EOL> return <NUM_LIT:0> <EOL> return len ( self . deleted ) <EOL> def delete_document ( self , docnum , delete = True ) : <EOL> """<STR_LIT>""" <EOL> if delete : <EOL> if self . deleted is None : <EOL> self . deleted = set ( ) <EOL> self . deleted . add ( docnum ) <EOL> elif self . deleted is not None and docnum in self . deleted : <EOL> self . deleted . clear ( docnum ) <EOL> def is_deleted ( self , docnum ) : <EOL> """<STR_LIT>""" <EOL> if self . deleted is None : <EOL> return False <EOL> return docnum in self . deleted </s>
