<s> '''<STR_LIT>''' <EOL> from __future__ import absolute_import , division , print_function <EOL> from itertools import chain , imap , product <EOL> import logging <EOL> import nilsimsa <EOL> from dossier . fc import FeatureCollection as FC , StringCounter <EOL> from dossier . web . interface import Filter <EOL> logger = logging . getLogger ( __name__ ) <EOL> class already_labeled ( Filter ) : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , label_store ) : <EOL> super ( already_labeled , self ) . __init__ ( ) <EOL> self . label_store = label_store <EOL> def create_predicate ( self ) : <EOL> labeled = self . label_store . directly_connected ( self . query_content_id ) <EOL> labeled_cids = set ( lab . other ( self . query_content_id ) for lab in labeled ) <EOL> return lambda ( cid , _ ) : cid not in labeled_cids <EOL> class geotime ( Filter ) : <EOL> '''<STR_LIT>''' <EOL> param_schema = dict ( Filter . param_schema , ** { <EOL> '<STR_LIT>' : { '<STR_LIT:type>' : '<STR_LIT:float>' , '<STR_LIT>' : - <NUM_LIT> , '<STR_LIT>' : <NUM_LIT> } , <EOL> '<STR_LIT>' : { '<STR_LIT:type>' : '<STR_LIT:float>' , '<STR_LIT>' : - <NUM_LIT> , '<STR_LIT>' : <NUM_LIT> } , <EOL> '<STR_LIT>' : { '<STR_LIT:type>' : '<STR_LIT:float>' , '<STR_LIT>' : - <NUM_LIT> , '<STR_LIT>' : <NUM_LIT> } , <EOL> '<STR_LIT>' : { '<STR_LIT:type>' : '<STR_LIT:float>' , '<STR_LIT>' : - <NUM_LIT> , '<STR_LIT>' : <NUM_LIT> } , <EOL> '<STR_LIT>' : { '<STR_LIT:type>' : '<STR_LIT:float>' , '<STR_LIT>' : - <NUM_LIT> , '<STR_LIT>' : <NUM_LIT> } , <EOL> '<STR_LIT>' : { '<STR_LIT:type>' : '<STR_LIT:float>' , '<STR_LIT>' : - <NUM_LIT> , '<STR_LIT>' : <NUM_LIT> } , <EOL> '<STR_LIT>' : { '<STR_LIT:type>' : '<STR_LIT:float>' , '<STR_LIT>' : <NUM_LIT:0> , '<STR_LIT>' : ( <NUM_LIT:2> ** <NUM_LIT:32> ) - <NUM_LIT:1> } , <EOL> '<STR_LIT>' : { '<STR_LIT:type>' : '<STR_LIT:float>' , '<STR_LIT>' : <NUM_LIT:0> , '<STR_LIT>' : ( <NUM_LIT:2> ** <NUM_LIT:32> ) - <NUM_LIT:1> } , <EOL> } ) <EOL> def __init__ ( self , <EOL> geotime_feature_name = FC . GEOCOORDS_PREFIX + '<STR_LIT>' ) : <EOL> super ( geotime , self ) . __init__ ( ) <EOL> self . geotime_feature_name = geotime_feature_name <EOL> def create_predicate ( self ) : <EOL> dim_names = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:time>' ] <EOL> min_dim = [ self . params [ '<STR_LIT>' + dname ] for dname in dim_names ] <EOL> max_dim = [ self . params [ '<STR_LIT>' + dname ] for dname in dim_names ] <EOL> if all ( x is None for x in chain ( min_dim , max_dim ) ) : <EOL> return lambda _ : True <EOL> def in_bbox ( coords ) : <EOL> '''<STR_LIT>''' <EOL> for d in range ( <NUM_LIT:4> ) : <EOL> if min_dim [ d ] is not None : <EOL> if coords [ d ] is None or coords [ d ] < min_dim [ d ] : <EOL> return False <EOL> if max_dim [ d ] is not None : <EOL> if coords [ d ] is None or coords [ d ] > max_dim [ d ] : <EOL> return False <EOL> return True <EOL> def pred ( ( cid , fc ) ) : <EOL> feature = fc . get ( self . geotime_feature_name , { } ) <EOL> return any ( any ( imap ( in_bbox , coords ) ) <EOL> for _ , coords in feature . iteritems ( ) ) <EOL> return pred <EOL> class nilsimsa_near_duplicates ( Filter ) : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , label_store , store , <EOL> nilsimsa_feature_name = '<STR_LIT>' , threshold = <NUM_LIT> ) : <EOL> self . label_store = label_store <EOL> self . store = store <EOL> self . nilsimsa_feature_name = nilsimsa_feature_name <EOL> self . threshold = threshold <EOL> logger . info ( '<STR_LIT>' , <EOL> nilsimsa_feature_name , threshold ) <EOL> def create_predicate ( self ) : <EOL> query_fc = self . get_query_fc ( ) <EOL> sim_feature = get_string_counter ( query_fc , self . nilsimsa_feature_name ) <EOL> accumulator = dict ( ) <EOL> if sim_feature : <EOL> for nhash in sim_feature : <EOL> accumulator [ nhash ] = self . query_content_id <EOL> def accumulating_predicate ( ( content_id , fc ) ) : <EOL> sim_feature = get_string_counter ( fc , self . nilsimsa_feature_name ) <EOL> if not sim_feature : <EOL> return True <EOL> for nhash in sim_feature : <EOL> if nhash in accumulator : <EOL> return False <EOL> for hash1 , hash2 in product ( sim_feature , accumulator ) : <EOL> score = nilsimsa . compare_digests ( hash1 , hash2 , <EOL> threshold = self . threshold ) <EOL> score /= <NUM_LIT> <EOL> if score > self . threshold : <EOL> return False <EOL> for nhash in sim_feature : <EOL> accumulator [ nhash ] = content_id <EOL> return True <EOL> return accumulating_predicate <EOL> def get_query_fc ( self ) : <EOL> return self . store . get ( self . query_content_id ) <EOL> def get_string_counter ( fc , feature_name ) : <EOL> '''<STR_LIT>''' <EOL> if feature_name not in fc : <EOL> feature = fc . get ( FC . DISPLAY_PREFIX + feature_name ) <EOL> else : <EOL> feature = fc . get ( feature_name ) <EOL> if isinstance ( feature , StringCounter ) : <EOL> return feature <EOL> else : <EOL> return None </s>
