<s> from future . utils import viewitems , viewvalues <EOL> from collections import defaultdict <EOL> import collections <EOL> import itertools <EOL> import logging <EOL> import time <EOL> logger = logging . getLogger ( __name__ ) <EOL> class Blocker : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , predicates ) : <EOL> self . predicates = predicates <EOL> self . index_fields = defaultdict ( lambda : defaultdict ( list ) ) <EOL> for full_predicate in predicates : <EOL> for predicate in full_predicate : <EOL> if hasattr ( predicate , '<STR_LIT:index>' ) : <EOL> self . index_fields [ predicate . field ] [ predicate . type ] . append ( predicate ) <EOL> def __call__ ( self , records ) : <EOL> start_time = time . clock ( ) <EOL> predicates = [ ( '<STR_LIT::>' + str ( i ) , predicate ) <EOL> for i , predicate <EOL> in enumerate ( self . predicates ) ] <EOL> for i , record in enumerate ( records ) : <EOL> record_id , instance = record <EOL> for pred_id , predicate in predicates : <EOL> block_keys = predicate ( instance ) <EOL> for block_key in block_keys : <EOL> yield block_key + pred_id , record_id <EOL> if i and i % <NUM_LIT> == <NUM_LIT:0> : <EOL> logger . info ( '<STR_LIT>' , <EOL> { '<STR_LIT>' : i , <EOL> '<STR_LIT>' : time . clock ( ) - start_time } ) <EOL> def resetIndices ( self ) : <EOL> for index_type in self . index_fields . values ( ) : <EOL> for predicates in index_type . values ( ) : <EOL> for predicate in predicates : <EOL> predicate . index = None <EOL> if hasattr ( predicate , '<STR_LIT>' ) : <EOL> predicate . canopy = { } <EOL> def index ( self , data , field ) : <EOL> '''<STR_LIT>''' <EOL> indices = extractIndices ( self . index_fields [ field ] ) <EOL> for doc in data : <EOL> if doc : <EOL> for _ , index , preprocess in indices : <EOL> index . index ( preprocess ( doc ) ) <EOL> for index_type , index , _ in indices : <EOL> index . initSearch ( ) <EOL> for predicate in self . index_fields [ field ] [ index_type ] : <EOL> logger . info ( "<STR_LIT>" , str ( predicate ) ) <EOL> predicate . index = index <EOL> def unindex ( self , data , field ) : <EOL> '''<STR_LIT>''' <EOL> indices = extractIndices ( self . index_fields [ field ] ) <EOL> for doc in data : <EOL> if doc : <EOL> for _ , index , preprocess in indices : <EOL> index . unindex ( preprocess ( doc ) ) <EOL> for index_type , index , _ in indices : <EOL> index . _index . initSearch ( ) <EOL> for predicate in self . index_fields [ field ] [ index_type ] : <EOL> logger . info ( "<STR_LIT>" , str ( predicate ) ) <EOL> predicate . index = index <EOL> def indexAll ( self , data_d ) : <EOL> for field in self . index_fields : <EOL> unique_fields = { record [ field ] <EOL> for record <EOL> in viewvalues ( data_d ) <EOL> if record [ field ] } <EOL> self . index ( unique_fields , field ) <EOL> def extractIndices ( index_fields ) : <EOL> indices = [ ] <EOL> for index_type , predicates in index_fields . items ( ) : <EOL> predicate = predicates [ <NUM_LIT:0> ] <EOL> index = predicate . index <EOL> preprocess = predicate . preprocess <EOL> if predicate . index is None : <EOL> index = predicate . initIndex ( ) <EOL> indices . append ( ( index_type , index , preprocess ) ) <EOL> return indices </s>
