<s> from __future__ import with_statement <EOL> import threading <EOL> import time <EOL> from whoosh . store import LockError <EOL> from whoosh . util import abstractmethod , synchronized <EOL> class IndexingError ( Exception ) : <EOL> pass <EOL> class IndexWriter ( object ) : <EOL> """<STR_LIT>""" <EOL> def __enter__ ( self ) : <EOL> return self <EOL> def __exit__ ( self , exc_type , exc_val , exc_tb ) : <EOL> if exc_type : <EOL> self . cancel ( ) <EOL> else : <EOL> self . commit ( ) <EOL> def add_field ( self , fieldname , fieldtype , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> self . schema . add ( fieldname , fieldtype , ** kwargs ) <EOL> def remove_field ( self , fieldname , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> self . schema . remove ( fieldname , ** kwargs ) <EOL> @ abstractmethod <EOL> def reader ( self , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError <EOL> def searcher ( self , ** kwargs ) : <EOL> from whoosh . searching import Searcher <EOL> return Searcher ( self . reader ( ) , ** kwargs ) <EOL> def delete_by_term ( self , fieldname , text , searcher = None ) : <EOL> """<STR_LIT>""" <EOL> from whoosh . query import Term <EOL> q = Term ( fieldname , text ) <EOL> return self . delete_by_query ( q , searcher = searcher ) <EOL> def delete_by_query ( self , q , searcher = None ) : <EOL> """<STR_LIT>""" <EOL> if searcher : <EOL> s = searcher <EOL> else : <EOL> s = self . searcher ( ) <EOL> try : <EOL> count = <NUM_LIT:0> <EOL> for docnum in s . docs_for_query ( q ) : <EOL> if not self . is_deleted ( docnum ) : <EOL> self . delete_document ( docnum ) <EOL> count += <NUM_LIT:1> <EOL> finally : <EOL> if not searcher : <EOL> s . close ( ) <EOL> return count <EOL> @ abstractmethod <EOL> def delete_document ( self , docnum , delete = True ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError <EOL> @ abstractmethod <EOL> def add_document ( self , ** fields ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError <EOL> def _doc_boost ( self , fields , default = <NUM_LIT:1.0> ) : <EOL> if "<STR_LIT>" in fields : <EOL> return float ( fields [ "<STR_LIT>" ] ) <EOL> else : <EOL> return default <EOL> def _field_boost ( self , fields , fieldname , default = <NUM_LIT:1.0> ) : <EOL> boostkw = "<STR_LIT>" % fieldname <EOL> if boostkw in fields : <EOL> return float ( fields [ boostkw ] ) <EOL> else : <EOL> return default <EOL> def _unique_fields ( self , fields ) : <EOL> unique_fields = [ name for name , field in self . schema . items ( ) <EOL> if name in fields and field . unique ] <EOL> return unique_fields <EOL> def update_document ( self , ** fields ) : <EOL> """<STR_LIT>""" <EOL> unique_fields = self . _unique_fields ( fields ) <EOL> if unique_fields : <EOL> with self . searcher ( ) as s : <EOL> for docnum in s . _find_unique ( [ ( name , fields [ name ] ) <EOL> for name in unique_fields ] ) : <EOL> self . delete_document ( docnum ) <EOL> self . add_document ( ** fields ) <EOL> def commit ( self ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> def cancel ( self ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> class AsyncWriter ( threading . Thread , IndexWriter ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , index , delay = <NUM_LIT> , writerargs = None ) : <EOL> """<STR_LIT>""" <EOL> threading . Thread . __init__ ( self ) <EOL> self . running = False <EOL> self . index = index <EOL> self . writerargs = writerargs or { } <EOL> self . delay = delay <EOL> self . events = [ ] <EOL> try : <EOL> self . writer = self . index . writer ( ** self . writerargs ) <EOL> except LockError : <EOL> self . writer = None <EOL> def reader ( self ) : <EOL> return self . index . reader ( ) <EOL> def searcher ( self , ** kwargs ) : <EOL> from whoosh . searching import Searcher <EOL> return Searcher ( self . reader ( ) , fromindex = self . index , ** kwargs ) <EOL> def _record ( self , method , args , kwargs ) : <EOL> if self . writer : <EOL> getattr ( self . writer , method ) ( * args , ** kwargs ) <EOL> else : <EOL> self . events . append ( ( method , args , kwargs ) ) <EOL> def run ( self ) : <EOL> self . running = True <EOL> writer = self . writer <EOL> while writer is None : <EOL> try : <EOL> writer = self . index . writer ( ** self . writerargs ) <EOL> except LockError : <EOL> time . sleep ( self . delay ) <EOL> for method , args , kwargs in self . events : <EOL> getattr ( writer , method ) ( * args , ** kwargs ) <EOL> writer . commit ( * self . commitargs , ** self . commitkwargs ) <EOL> def delete_document ( self , * args , ** kwargs ) : <EOL> self . _record ( "<STR_LIT>" , args , kwargs ) <EOL> def add_document ( self , * args , ** kwargs ) : <EOL> self . _record ( "<STR_LIT>" , args , kwargs ) <EOL> def update_document ( self , * args , ** kwargs ) : <EOL> self . _record ( "<STR_LIT>" , args , kwargs ) <EOL> def add_field ( self , * args , ** kwargs ) : <EOL> self . _record ( "<STR_LIT>" , args , kwargs ) <EOL> def remove_field ( self , * args , ** kwargs ) : <EOL> self . _record ( "<STR_LIT>" , args , kwargs ) <EOL> def delete_by_term ( self , * args , ** kwargs ) : <EOL> self . _record ( "<STR_LIT>" , args , kwargs ) <EOL> def commit ( self , * args , ** kwargs ) : <EOL> if self . writer : <EOL> self . writer . commit ( * args , ** kwargs ) <EOL> else : <EOL> self . commitargs , self . commitkwargs = args , kwargs <EOL> self . start ( ) <EOL> def cancel ( self , * args , ** kwargs ) : <EOL> if self . writer : <EOL> self . writer . cancel ( * args , ** kwargs ) <EOL> class BufferedWriter ( IndexWriter ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , index , period = <NUM_LIT> , limit = <NUM_LIT:10> , writerargs = None , <EOL> commitargs = None , tempixclass = None ) : <EOL> """<STR_LIT>""" <EOL> self . index = index <EOL> self . period = period <EOL> self . limit = limit <EOL> self . writerargs = writerargs or { } <EOL> self . commitargs = commitargs or { } <EOL> self . _sync_lock = threading . RLock ( ) <EOL> self . _write_lock = threading . Lock ( ) <EOL> if tempixclass is None : <EOL> from whoosh . ramindex import RamIndex as tempixclass <EOL> self . tempixclass = tempixclass <EOL> self . writer = None <EOL> self . base = self . index . doc_count_all ( ) <EOL> self . bufferedcount = <NUM_LIT:0> <EOL> self . commitcount = <NUM_LIT:0> <EOL> self . ramindex = self . _create_ramindex ( ) <EOL> if self . period : <EOL> self . timer = threading . Timer ( self . period , self . commit ) <EOL> def __del__ ( self ) : <EOL> if hasattr ( self , "<STR_LIT>" ) and self . writer : <EOL> if not self . writer . is_closed : <EOL> self . writer . cancel ( ) <EOL> del self . writer <EOL> def _create_ramindex ( self ) : <EOL> return self . tempixclass ( self . index . schema ) <EOL> def _get_writer ( self ) : <EOL> if self . writer is None : <EOL> self . writer = self . index . writer ( ** self . writerargs ) <EOL> self . schema = self . writer . schema <EOL> self . base = self . index . doc_count_all ( ) <EOL> self . bufferedcount = <NUM_LIT:0> <EOL> return self . writer <EOL> @ synchronized <EOL> def reader ( self , ** kwargs ) : <EOL> from whoosh . reading import MultiReader <EOL> writer = self . _get_writer ( ) <EOL> ramreader = self . ramindex <EOL> if self . index . is_empty ( ) : <EOL> return ramreader <EOL> else : <EOL> reader = writer . reader ( ** kwargs ) <EOL> if reader . is_atomic ( ) : <EOL> reader = MultiReader ( [ reader , ramreader ] ) <EOL> else : <EOL> reader . add_reader ( ramreader ) <EOL> return reader <EOL> def searcher ( self , ** kwargs ) : <EOL> from whoosh . searching import Searcher <EOL> return Searcher ( self . reader ( ) , fromindex = self . index , ** kwargs ) <EOL> def close ( self ) : <EOL> self . commit ( restart = False ) <EOL> def commit ( self , restart = True ) : <EOL> if self . period : <EOL> self . timer . cancel ( ) <EOL> with self . _sync_lock : <EOL> oldramindex = self . ramindex <EOL> self . ramindex = self . _create_ramindex ( ) <EOL> with self . _write_lock : <EOL> if self . bufferedcount : <EOL> self . _get_writer ( ) . add_reader ( oldramindex . reader ( ) ) <EOL> if self . writer : <EOL> self . writer . commit ( ** self . commitargs ) <EOL> self . writer = None <EOL> self . commitcount += <NUM_LIT:1> <EOL> if restart : <EOL> if self . period : <EOL> self . timer = threading . Timer ( self . period , self . commit ) <EOL> def add_reader ( self , reader ) : <EOL> with self . _write_lock : <EOL> self . _get_writer ( ) . add_reader ( reader ) <EOL> def add_document ( self , ** fields ) : <EOL> with self . _sync_lock : <EOL> self . ramindex . add_document ( ** fields ) <EOL> self . bufferedcount += <NUM_LIT:1> <EOL> if self . bufferedcount >= self . limit : <EOL> self . commit ( ) <EOL> @ synchronized <EOL> def update_document ( self , ** fields ) : <EOL> self . _get_writer ( ) <EOL> super ( BufferedWriter , self ) . update_document ( ** fields ) <EOL> @ synchronized <EOL> def delete_document ( self , docnum , delete = True ) : <EOL> if docnum < self . base : <EOL> return self . _get_writer ( ) . delete_document ( docnum , delete = delete ) <EOL> else : <EOL> return self . ramindex . delete_document ( docnum - self . base , <EOL> delete = delete ) <EOL> @ synchronized <EOL> def is_deleted ( self , docnum ) : <EOL> if docnum < self . base : <EOL> return self . writer . is_deleted ( docnum ) <EOL> else : <EOL> return self . ramindex . is_deleted ( docnum - self . base ) <EOL> BatchWriter = BufferedWriter </s>
