<s> import time <EOL> import random <EOL> import numpy as np <EOL> import time <EOL> from scipy . sparse import csr_matrix <EOL> from sklearn . datasets import fetch_20newsgroups <EOL> from sklearn . neighbors import LSHForest <EOL> from sklearn . feature_extraction import DictVectorizer <EOL> import pysparnn <EOL> class PySparNNTextSearch : <EOL> def __init__ ( self , docs , datas , matrix_size = <NUM_LIT> ) : <EOL> self . dv = DictVectorizer ( ) <EOL> dicts = [ ] <EOL> for d in docs : <EOL> dicts . append ( dict ( [ ( w , <NUM_LIT:1> ) for w in d ] ) ) <EOL> self . dv . fit ( dicts ) <EOL> features = csr_matrix ( self . dv . transform ( dicts ) , dtype = int ) <EOL> self . cp = pysparnn . ClusterIndex ( features , datas , pysparnn . matrix_distance . UnitCosineDistance , matrix_size = matrix_size ) <EOL> def search ( self , docs , k = <NUM_LIT:1> , min_distance = None , max_distance = None , k_clusters = <NUM_LIT:1> , return_distance = False ) : <EOL> dicts = [ ] <EOL> for d in docs : <EOL> dicts . append ( dict ( [ ( w , <NUM_LIT:1> ) for w in d ] ) ) <EOL> features = csr_matrix ( self . dv . transform ( dicts ) , dtype = int ) <EOL> return self . cp . search ( features , k = k , min_distance = min_distance , max_distance = max_distance , <EOL> k_clusters = k_clusters , return_distance = return_distance ) <EOL> class LSHForestSearch : <EOL> def __init__ ( self , docs ) : <EOL> self . lshf = LSHForest ( n_estimators = <NUM_LIT:1> , n_candidates = <NUM_LIT:1> , <EOL> n_neighbors = <NUM_LIT:1> ) <EOL> self . dv = DictVectorizer ( ) <EOL> dicts = [ ] <EOL> for d in docs : <EOL> dicts . append ( dict ( [ ( w , <NUM_LIT:1> ) for w in d ] ) ) <EOL> self . dv . fit ( dicts ) <EOL> features = self . dv . transform ( dicts ) <EOL> self . lshf . fit ( features ) <EOL> def search ( self , docs ) : <EOL> dicts = [ ] <EOL> for d in docs : <EOL> dicts . append ( dict ( [ ( w , <NUM_LIT:1> ) for w in d ] ) ) <EOL> features = self . dv . transform ( dicts ) <EOL> return self . lshf . kneighbors ( features , return_distance = False ) <EOL> def accuracy ( result , truth ) : <EOL> ret = [ ] <EOL> for r , t in zip ( result , truth ) : <EOL> ret . append ( <NUM_LIT:1> if t in r else <NUM_LIT:0> ) <EOL> return np . array ( ret ) <EOL> def time_it ( search_index , docs , query_index ) : <EOL> query_docs = [ ] <EOL> for i in query_index : <EOL> query_docs . append ( docs [ i ] ) <EOL> t0 = time . time ( ) <EOL> neighbors = search_index . search ( query_docs ) <EOL> delta = time . time ( ) - t0 <EOL> return delta , accuracy ( neighbors , query_index ) . mean ( ) <EOL> def identity_benchmark ( search_index , docs , n_trials = <NUM_LIT:1000> , docs_per_query = <NUM_LIT:50> ) : <EOL> times = [ ] <EOL> accuracys = [ ] <EOL> for i in range ( n_trials ) : <EOL> query_index = random . sample ( range ( len ( docs ) ) , docs_per_query ) <EOL> time , accuracy = time_it ( search_index , docs , query_index ) <EOL> time = time / docs_per_query <EOL> times . append ( time ) <EOL> accuracys . append ( accuracy ) <EOL> return np . median ( times ) , np . median ( accuracys ) </s>
