<s> '''<STR_LIT>''' <EOL> import struct <EOL> import numpy as np <EOL> class bBitMinHash ( object ) : <EOL> '''<STR_LIT>''' <EOL> __slots__ = ( '<STR_LIT>' , '<STR_LIT:b>' , '<STR_LIT:r>' , '<STR_LIT>' ) <EOL> _serial_fmt_params = '<STR_LIT>' <EOL> _serial_fmt_block = '<STR_LIT>' <EOL> def __init__ ( self , minhash , b = <NUM_LIT:1> , r = <NUM_LIT:0.0> ) : <EOL> '''<STR_LIT>''' <EOL> b = int ( b ) <EOL> r = float ( r ) <EOL> if b > <NUM_LIT:32> or b < <NUM_LIT:0> : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> if r > <NUM_LIT:1.0> : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> bmask = ( <NUM_LIT:1> << b ) - <NUM_LIT:1> <EOL> self . hashvalues = np . bitwise_and ( minhash . hashvalues , bmask ) . astype ( np . uint32 ) <EOL> self . seed = minhash . seed <EOL> self . b = b <EOL> self . r = r <EOL> def __eq__ ( self , other ) : <EOL> '''<STR_LIT>''' <EOL> return self . seed == other . seed and self . b == other . b and self . r == other . r and np . array_equal ( self . hashvalues , other . hashvalues ) <EOL> def jaccard ( self , other ) : <EOL> '''<STR_LIT>''' <EOL> if self . b != other . b : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> if self . seed != other . seed : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> intersection = np . count_nonzero ( self . hashvalues == other . hashvalues ) <EOL> raw_est = float ( intersection ) / float ( self . hashvalues . size ) <EOL> a1 = self . _calc_a ( self . r , self . b ) <EOL> a2 = self . _calc_a ( other . r , other . b ) <EOL> c1 , c2 = self . _calc_c ( a1 , a2 , self . r , other . r ) <EOL> return ( raw_est - c1 ) / ( <NUM_LIT:1> - c2 ) <EOL> def bytesize ( self ) : <EOL> '''<STR_LIT>''' <EOL> return self . _bytesize ( ) [ - <NUM_LIT:1> ] <EOL> def __getstate__ ( self ) : <EOL> '''<STR_LIT>''' <EOL> slot_size , n , num_blocks , total = self . _bytesize ( ) <EOL> buffer = bytearray ( total ) <EOL> blocks = np . zeros ( ( num_blocks , ) , dtype = np . uint64 ) <EOL> for i in range ( num_blocks ) : <EOL> start = i * n <EOL> hvs = self . hashvalues [ start : start + n ] <EOL> for j , hv in enumerate ( hvs ) : <EOL> blocks [ i ] |= np . uint64 ( hv << ( n - <NUM_LIT:1> - j ) * slot_size ) <EOL> fmt = self . _serial_fmt_params + "<STR_LIT>" % ( num_blocks , self . _serial_fmt_block ) <EOL> struct . pack_into ( fmt , buffer , <NUM_LIT:0> , self . seed , self . b , self . r , self . hashvalues . size , * blocks ) <EOL> return buffer <EOL> def __setstate__ ( self , buf ) : <EOL> '''<STR_LIT>''' <EOL> try : <EOL> self . seed , self . b , self . r , num_perm = struct . unpack_from ( self . _serial_fmt_params , buf , <NUM_LIT:0> ) <EOL> except TypeError : <EOL> self . seed , self . b , self . r , num_perm = struct . unpack_from ( self . _serial_fmt_params , buffer ( buf ) , <NUM_LIT:0> ) <EOL> offset = struct . calcsize ( self . _serial_fmt_params ) <EOL> self . hashvalues = np . zeros ( ( num_perm , ) , dtype = np . uint32 ) <EOL> slot_size , n , num_blocks , total = self . _bytesize ( ) <EOL> fmt = "<STR_LIT>" % ( num_blocks , self . _serial_fmt_block ) <EOL> try : <EOL> blocks = struct . unpack_from ( fmt , buf , offset ) <EOL> except TypeError : <EOL> blocks = struct . unpack_from ( fmt , buffer ( buf ) , offset ) <EOL> mask = ( <NUM_LIT:1> << slot_size ) - <NUM_LIT:1> <EOL> for i in range ( num_blocks ) : <EOL> start = i * n <EOL> for j , _ in enumerate ( self . hashvalues [ start : start + n ] ) : <EOL> hv = ( blocks [ i ] >> ( n - <NUM_LIT:1> - j ) * slot_size ) & mask <EOL> self . hashvalues [ start + j ] = np . uint32 ( hv ) <EOL> def _calc_a ( self , r , b ) : <EOL> '''<STR_LIT>''' <EOL> if r == <NUM_LIT:0.0> : <EOL> return <NUM_LIT:1.0> / ( <NUM_LIT:1> << b ) <EOL> return r * ( <NUM_LIT:1> - r ) ** ( <NUM_LIT:2> ** b - <NUM_LIT:1> ) / ( <NUM_LIT:1> - ( <NUM_LIT:1> - r ) ** ( <NUM_LIT:2> * b ) ) <EOL> def _calc_c ( self , a1 , a2 , r1 , r2 ) : <EOL> '''<STR_LIT>''' <EOL> if r1 == <NUM_LIT:0.0> and r2 == <NUM_LIT:0.0> : <EOL> return a1 , a2 <EOL> div = <NUM_LIT:1> / ( r1 + r2 ) <EOL> c1 = ( a1 * r2 + a2 * r1 ) * div <EOL> c2 = ( a1 * r1 + a2 * r2 ) * div <EOL> return c1 , c2 <EOL> def _find_slot_size ( self , b ) : <EOL> if b == <NUM_LIT:1> : <EOL> return <NUM_LIT:1> <EOL> if b == <NUM_LIT:2> : <EOL> return <NUM_LIT:2> <EOL> if b <= <NUM_LIT:4> : <EOL> return <NUM_LIT:4> <EOL> if b <= <NUM_LIT:8> : <EOL> return <NUM_LIT:8> <EOL> if b <= <NUM_LIT:16> : <EOL> return <NUM_LIT:16> <EOL> if b <= <NUM_LIT:32> : <EOL> return <NUM_LIT:32> <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> def _bytesize ( self ) : <EOL> block_size = struct . calcsize ( self . _serial_fmt_block ) <EOL> slot_size = self . _find_slot_size ( self . b ) <EOL> num_slots_per_block = int ( block_size * <NUM_LIT:8> / slot_size ) <EOL> num_blocks = int ( np . ceil ( float ( self . hashvalues . size ) / num_slots_per_block ) ) <EOL> total = struct . calcsize ( self . _serial_fmt_params + "<STR_LIT>" % ( num_blocks , self . _serial_fmt_block ) ) <EOL> return slot_size , num_slots_per_block , num_blocks , total </s>
