<s> import zlib , bz2 <EOL> import math , json , re <EOL> def simhash ( x ) : <EOL> import simhash <EOL> return simhash . simhash ( x ) <EOL> def entropy ( data ) : <EOL> entropy = <NUM_LIT:0.0> <EOL> if len ( data ) == <NUM_LIT:0> : <EOL> return entropy <EOL> for x in range ( <NUM_LIT> ) : <EOL> p_x = float ( data . count ( chr ( x ) ) ) / len ( data ) <EOL> if p_x > <NUM_LIT:0> : <EOL> entropy += - p_x * math . log ( p_x , <NUM_LIT:2> ) <EOL> return entropy <EOL> try : <EOL> from ctypes import cdll , c_float , c_double , c_int , c_uint , c_void_p , Structure , addressof , cast , c_size_t <EOL> class LIBSIMILARITY_T ( Structure ) : <EOL> _fields_ = [ ( "<STR_LIT>" , c_void_p ) , <EOL> ( "<STR_LIT>" , c_size_t ) , <EOL> ( "<STR_LIT>" , c_void_p ) , <EOL> ( "<STR_LIT>" , c_size_t ) , <EOL> ( "<STR_LIT>" , c_size_t ) , <EOL> ( "<STR_LIT>" , c_size_t ) , <EOL> ( "<STR_LIT>" , c_float ) , <EOL> ] <EOL> def new_zero_native ( ) : <EOL> return c_size_t ( <NUM_LIT:0> ) <EOL> NATIVE_LIB = True <EOL> except : <EOL> NATIVE_LIB = False <EOL> def new_zero_python ( ) : <EOL> return <NUM_LIT:0> <EOL> ZLIB_COMPRESS = <NUM_LIT:0> <EOL> BZ2_COMPRESS = <NUM_LIT:1> <EOL> SMAZ_COMPRESS = <NUM_LIT:2> <EOL> LZMA_COMPRESS = <NUM_LIT:3> <EOL> XZ_COMPRESS = <NUM_LIT:4> <EOL> SNAPPY_COMPRESS = <NUM_LIT:5> <EOL> VCBLOCKSORT_COMPRESS = <NUM_LIT:6> <EOL> H_COMPRESSOR = { "<STR_LIT>" : BZ2_COMPRESS , <EOL> "<STR_LIT>" : ZLIB_COMPRESS , <EOL> "<STR_LIT>" : LZMA_COMPRESS , <EOL> "<STR_LIT>" : XZ_COMPRESS , <EOL> "<STR_LIT>" : SNAPPY_COMPRESS , <EOL> } <EOL> HR_COMPRESSOR = { <EOL> BZ2_COMPRESS : "<STR_LIT>" , <EOL> ZLIB_COMPRESS : "<STR_LIT>" , <EOL> LZMA_COMPRESS : "<STR_LIT>" , <EOL> XZ_COMPRESS : "<STR_LIT>" , <EOL> SNAPPY_COMPRESS : "<STR_LIT>" , <EOL> } <EOL> class SIMILARITYBase ( object ) : <EOL> def __init__ ( self , native_lib = False ) : <EOL> self . ctype = ZLIB_COMPRESS <EOL> self . __caches = { <EOL> ZLIB_COMPRESS : { } , <EOL> BZ2_COMPRESS : { } , <EOL> SMAZ_COMPRESS : { } , <EOL> LZMA_COMPRESS : { } , <EOL> XZ_COMPRESS : { } , <EOL> SNAPPY_COMPRESS : { } , <EOL> VCBLOCKSORT_COMPRESS : { } , <EOL> } <EOL> self . __rcaches = { <EOL> ZLIB_COMPRESS : { } , <EOL> BZ2_COMPRESS : { } , <EOL> SMAZ_COMPRESS : { } , <EOL> LZMA_COMPRESS : { } , <EOL> XZ_COMPRESS : { } , <EOL> SNAPPY_COMPRESS : { } , <EOL> VCBLOCKSORT_COMPRESS : { } , <EOL> } <EOL> self . __ecaches = { } <EOL> self . level = <NUM_LIT:9> <EOL> if native_lib == True : <EOL> self . new_zero = new_zero_native <EOL> else : <EOL> self . new_zero = new_zero_python <EOL> def set_level ( self , level ) : <EOL> self . level = level <EOL> def get_in_caches ( self , s ) : <EOL> try : <EOL> return self . __caches [ self . ctype ] [ zlib . adler32 ( s ) ] <EOL> except KeyError : <EOL> return self . new_zero ( ) <EOL> def get_in_rcaches ( self , s1 , s2 ) : <EOL> try : <EOL> return self . __rcaches [ self . ctype ] [ zlib . adler32 ( s1 + s2 ) ] <EOL> except KeyError : <EOL> try : <EOL> return self . __rcaches [ self . ctype ] [ zlib . adler32 ( s2 + s1 ) ] <EOL> except KeyError : <EOL> return - <NUM_LIT:1> , - <NUM_LIT:1> <EOL> def add_in_caches ( self , s , v ) : <EOL> h = zlib . adler32 ( s ) <EOL> if h not in self . __caches [ self . ctype ] : <EOL> self . __caches [ self . ctype ] [ h ] = v <EOL> def add_in_rcaches ( self , s , v , r ) : <EOL> h = zlib . adler32 ( s ) <EOL> if h not in self . __rcaches [ self . ctype ] : <EOL> self . __rcaches [ self . ctype ] [ h ] = ( v , r ) <EOL> def clear_caches ( self ) : <EOL> for i in self . __caches : <EOL> self . __caches [ i ] = { } <EOL> def add_in_ecaches ( self , s , v , r ) : <EOL> h = zlib . adler32 ( s ) <EOL> if h not in self . __ecaches : <EOL> self . __ecaches [ h ] = ( v , r ) <EOL> def get_in_ecaches ( self , s1 ) : <EOL> try : <EOL> return self . __ecaches [ zlib . adler32 ( s1 ) ] <EOL> except KeyError : <EOL> return - <NUM_LIT:1> , - <NUM_LIT:1> <EOL> def __nb_caches ( self , caches ) : <EOL> nb = <NUM_LIT:0> <EOL> for i in caches : <EOL> nb += len ( caches [ i ] ) <EOL> return nb <EOL> def set_compress_type ( self , t ) : <EOL> self . ctype = t <EOL> def show ( self ) : <EOL> print "<STR_LIT>" , len ( self . __ecaches ) <EOL> print "<STR_LIT>" , self . __nb_caches ( self . __rcaches ) <EOL> print "<STR_LIT>" , self . __nb_caches ( self . __caches ) <EOL> class SIMILARITYNative ( SIMILARITYBase ) : <EOL> def __init__ ( self , path = "<STR_LIT>" ) : <EOL> super ( SIMILARITYNative , self ) . __init__ ( True ) <EOL> self . _u = cdll . LoadLibrary ( path ) <EOL> self . _u . compress . restype = c_uint <EOL> self . _u . ncd . restype = c_int <EOL> self . _u . ncs . restype = c_int <EOL> self . _u . cmid . restype = c_int <EOL> self . _u . entropy . restype = c_double <EOL> self . _u . levenshtein . restype = c_uint <EOL> self . _u . kolmogorov . restype = c_uint <EOL> self . _u . bennett . restype = c_double <EOL> self . _u . RDTSC . restype = c_double <EOL> self . __libsim_t = LIBSIMILARITY_T ( ) <EOL> self . set_compress_type ( ZLIB_COMPRESS ) <EOL> def raz ( self ) : <EOL> del self . _u <EOL> del self . __libsim_t <EOL> def compress ( self , s1 ) : <EOL> res = self . _u . compress ( self . level , cast ( s1 , c_void_p ) , len ( s1 ) ) <EOL> return res <EOL> def _sim ( self , s1 , s2 , func ) : <EOL> end , ret = self . get_in_rcaches ( s1 , s2 ) <EOL> if end != - <NUM_LIT:1> : <EOL> return end , ret <EOL> self . __libsim_t . orig = cast ( s1 , c_void_p ) <EOL> self . __libsim_t . size_orig = len ( s1 ) <EOL> self . __libsim_t . cmp = cast ( s2 , c_void_p ) <EOL> self . __libsim_t . size_cmp = len ( s2 ) <EOL> corig = self . get_in_caches ( s1 ) <EOL> ccmp = self . get_in_caches ( s2 ) <EOL> self . __libsim_t . corig = addressof ( corig ) <EOL> self . __libsim_t . ccmp = addressof ( ccmp ) <EOL> ret = func ( self . level , addressof ( self . __libsim_t ) ) <EOL> self . add_in_caches ( s1 , corig ) <EOL> self . add_in_caches ( s2 , ccmp ) <EOL> self . add_in_rcaches ( s1 + s2 , self . __libsim_t . res , ret ) <EOL> return self . __libsim_t . res , ret <EOL> def ncd ( self , s1 , s2 ) : <EOL> return self . _sim ( s1 , s2 , self . _u . ncd ) <EOL> def ncs ( self , s1 , s2 ) : <EOL> return self . _sim ( s1 , s2 , self . _u . ncs ) <EOL> def cmid ( self , s1 , s2 ) : <EOL> return self . _sim ( s1 , s2 , self . _u . cmid ) <EOL> def kolmogorov ( self , s1 ) : <EOL> ret = self . _u . kolmogorov ( self . level , cast ( s1 , c_void_p ) , len ( s1 ) ) <EOL> return ret , <NUM_LIT:0> <EOL> def bennett ( self , s1 ) : <EOL> ret = self . _u . bennett ( self . level , cast ( s1 , c_void_p ) , len ( s1 ) ) <EOL> return ret , <NUM_LIT:0> <EOL> def entropy ( self , s1 ) : <EOL> end , ret = self . get_in_ecaches ( s1 ) <EOL> if end != - <NUM_LIT:1> : <EOL> return end , ret <EOL> res = self . _u . entropy ( cast ( s1 , c_void_p ) , len ( s1 ) ) <EOL> self . add_in_ecaches ( s1 , res , <NUM_LIT:0> ) <EOL> return res , <NUM_LIT:0> <EOL> def RDTSC ( self ) : <EOL> return self . _u . RDTSC ( ) <EOL> def levenshtein ( self , s1 , s2 ) : <EOL> res = self . _u . levenshtein ( cast ( s1 , c_void_p ) , len ( s1 ) , cast ( s2 , c_void_p ) , len ( s2 ) ) <EOL> return res , <NUM_LIT:0> <EOL> def set_compress_type ( self , t ) : <EOL> self . ctype = t <EOL> self . _u . set_compress_type ( t ) <EOL> class SIMILARITYPython ( SIMILARITYBase ) : <EOL> def __init__ ( self ) : <EOL> super ( SIMILARITYPython , self ) . __init__ ( ) <EOL> def set_compress_type ( self , t ) : <EOL> self . ctype = t <EOL> if self . ctype != ZLIB_COMPRESS and self . ctype != BZ2_COMPRESS : <EOL> print "<STR_LIT>" % HR_COMPRESSOR [ t ] <EOL> self . ctype = ZLIB_COMPRESS <EOL> def compress ( self , s1 ) : <EOL> return len ( self . _compress ( s1 ) ) <EOL> def _compress ( self , s1 ) : <EOL> if self . ctype == ZLIB_COMPRESS : <EOL> return zlib . compress ( s1 , self . level ) <EOL> elif self . ctype == BZ2_COMPRESS : <EOL> return bz2 . compress ( s1 , self . level ) <EOL> def _sim ( self , s1 , s2 , func ) : <EOL> end , ret = self . get_in_rcaches ( s1 , s2 ) <EOL> if end != - <NUM_LIT:1> : <EOL> return end , ret <EOL> corig = self . get_in_caches ( s1 ) <EOL> ccmp = self . get_in_caches ( s2 ) <EOL> res , corig , ccmp , ret = func ( s1 , s2 , corig , ccmp ) <EOL> self . add_in_caches ( s1 , corig ) <EOL> self . add_in_caches ( s2 , ccmp ) <EOL> self . add_in_rcaches ( s1 + s2 , res , ret ) <EOL> return res , ret <EOL> def _ncd ( self , s1 , s2 , s1size = <NUM_LIT:0> , s2size = <NUM_LIT:0> ) : <EOL> if s1size == <NUM_LIT:0> : <EOL> s1size = self . compress ( s1 ) <EOL> if s2size == <NUM_LIT:0> : <EOL> s2size = self . compress ( s2 ) <EOL> s3size = self . compress ( s1 + s2 ) <EOL> smax = max ( s1size , s2size ) <EOL> smin = min ( s1size , s2size ) <EOL> res = ( abs ( s3size - smin ) ) / float ( smax ) <EOL> if res > <NUM_LIT:1.0> : <EOL> res = <NUM_LIT:1.0> <EOL> return res , s1size , s2size , <NUM_LIT:0> <EOL> def ncd ( self , s1 , s2 ) : <EOL> return self . _sim ( s1 , s2 , self . _ncd ) <EOL> def ncs ( self , s1 , s2 ) : <EOL> return self . _sim ( s1 , s2 , self . _u . ncs ) <EOL> def entropy ( self , s1 ) : <EOL> end , ret = self . get_in_ecaches ( s1 ) <EOL> if end != - <NUM_LIT:1> : <EOL> return end , ret <EOL> res = entropy ( s1 ) <EOL> self . add_in_ecaches ( s1 , res , <NUM_LIT:0> ) <EOL> return res , <NUM_LIT:0> <EOL> def levenshtein ( self , a , b ) : <EOL> "<STR_LIT>" <EOL> n , m = len ( a ) , len ( b ) <EOL> if n > m : <EOL> a , b = b , a <EOL> n , m = m , n <EOL> current = range ( n + <NUM_LIT:1> ) <EOL> for i in range ( <NUM_LIT:1> , m + <NUM_LIT:1> ) : <EOL> previous , current = current , [ i ] + [ <NUM_LIT:0> ] * n <EOL> for j in range ( <NUM_LIT:1> , n + <NUM_LIT:1> ) : <EOL> add , delete = previous [ j ] + <NUM_LIT:1> , current [ j - <NUM_LIT:1> ] + <NUM_LIT:1> <EOL> change = previous [ j - <NUM_LIT:1> ] <EOL> if a [ j - <NUM_LIT:1> ] != b [ i - <NUM_LIT:1> ] : <EOL> change = change + <NUM_LIT:1> <EOL> current [ j ] = min ( add , delete , change ) <EOL> return current [ n ] <EOL> class SIMILARITY ( object ) : <EOL> def __init__ ( self , path = "<STR_LIT>" , native_lib = True ) : <EOL> if native_lib == True and NATIVE_LIB == True : <EOL> try : <EOL> self . s = SIMILARITYNative ( path ) <EOL> except : <EOL> self . s = SIMILARITYPython ( ) <EOL> else : <EOL> self . s = SIMILARITYPython ( ) <EOL> def raz ( self ) : <EOL> return self . s . raz ( ) <EOL> def set_level ( self , level ) : <EOL> return self . s . set_level ( level ) <EOL> def compress ( self , s1 ) : <EOL> return self . s . compress ( s1 ) <EOL> def ncd ( self , s1 , s2 ) : <EOL> return self . s . ncd ( s1 , s2 ) <EOL> def ncs ( self , s1 , s2 ) : <EOL> return self . s . ncs ( s1 , s2 ) <EOL> def cmid ( self , s1 , s2 ) : <EOL> return self . s . cmid ( s1 , s2 ) <EOL> def kolmogorov ( self , s1 ) : <EOL> return self . s . kolmogorov ( s1 ) <EOL> def bennett ( self , s1 ) : <EOL> return self . s . bennett ( s1 ) <EOL> def entropy ( self , s1 ) : <EOL> return self . s . entropy ( s1 ) <EOL> def RDTSC ( self ) : <EOL> return self . s . RDTSC ( ) <EOL> def levenshtein ( self , s1 , s2 ) : <EOL> return self . s . levenshtein ( s1 , s2 ) <EOL> def set_compress_type ( self , t ) : <EOL> return self . s . set_compress_type ( t ) <EOL> def show ( self ) : <EOL> self . s . show ( ) <EOL> class DBFormat ( object ) : <EOL> def __init__ ( self , filename ) : <EOL> self . filename = filename <EOL> self . D = { } <EOL> fd = None <EOL> try : <EOL> with open ( self . filename , "<STR_LIT>" ) as fd : <EOL> self . D = json . load ( fd ) <EOL> except IOError : <EOL> print "<STR_LIT>" + filename <EOL> self . D = { } <EOL> self . H = { } <EOL> self . N = { } <EOL> for i in self . D : <EOL> self . H [ i ] = { } <EOL> for j in self . D [ i ] : <EOL> if j == "<STR_LIT>" : <EOL> self . N [ i ] = re . compile ( self . D [ i ] [ j ] ) <EOL> continue <EOL> self . H [ i ] [ j ] = { } <EOL> for k in self . D [ i ] [ j ] : <EOL> if isinstance ( self . D [ i ] [ j ] [ k ] , dict ) : <EOL> self . H [ i ] [ j ] [ k ] = set ( ) <EOL> for e in self . D [ i ] [ j ] [ k ] . keys ( ) : <EOL> self . H [ i ] [ j ] [ k ] . add ( long ( e ) ) <EOL> def add_name ( self , name , value ) : <EOL> if name not in self . D : <EOL> self . D [ name ] = { } <EOL> self . D [ name ] [ "<STR_LIT>" ] = value <EOL> def add_element ( self , name , sname , sclass , size , elem ) : <EOL> try : <EOL> if elem not in self . D [ name ] [ sname ] [ sclass ] : <EOL> self . D [ name ] [ sname ] [ sclass ] [ elem ] = size <EOL> self . D [ name ] [ sname ] [ "<STR_LIT>" ] += size <EOL> except KeyError : <EOL> if name not in self . D : <EOL> self . D [ name ] = { } <EOL> self . D [ name ] [ sname ] = { } <EOL> self . D [ name ] [ sname ] [ "<STR_LIT>" ] = <NUM_LIT:0> <EOL> self . D [ name ] [ sname ] [ sclass ] = { } <EOL> elif sname not in self . D [ name ] : <EOL> self . D [ name ] [ sname ] = { } <EOL> self . D [ name ] [ sname ] [ "<STR_LIT>" ] = <NUM_LIT:0> <EOL> self . D [ name ] [ sname ] [ sclass ] = { } <EOL> elif sclass not in self . D [ name ] [ sname ] : <EOL> self . D [ name ] [ sname ] [ sclass ] = { } <EOL> self . D [ name ] [ sname ] [ "<STR_LIT>" ] += size <EOL> self . D [ name ] [ sname ] [ sclass ] [ elem ] = size <EOL> def is_present ( self , elem ) : <EOL> for i in self . D : <EOL> if elem in self . D [ i ] : <EOL> return True , i <EOL> return False , None <EOL> def elems_are_presents ( self , elems ) : <EOL> ret = { } <EOL> info = { } <EOL> for i in self . H : <EOL> ret [ i ] = { } <EOL> info [ i ] = { } <EOL> for j in self . H [ i ] : <EOL> ret [ i ] [ j ] = { } <EOL> info [ i ] [ j ] = { } <EOL> for k in self . H [ i ] [ j ] : <EOL> val = [ self . H [ i ] [ j ] [ k ] . intersection ( elems ) , len ( self . H [ i ] [ j ] [ k ] ) , <NUM_LIT:0> , <NUM_LIT:0> ] <EOL> size = <NUM_LIT:0> <EOL> for z in val [ <NUM_LIT:0> ] : <EOL> size += self . D [ i ] [ j ] [ k ] [ str ( z ) ] <EOL> val [ <NUM_LIT:2> ] = ( float ( len ( val [ <NUM_LIT:0> ] ) ) / ( val [ <NUM_LIT:1> ] ) ) * <NUM_LIT:100> <EOL> val [ <NUM_LIT:3> ] = size <EOL> if val [ <NUM_LIT:3> ] != <NUM_LIT:0> : <EOL> ret [ i ] [ j ] [ k ] = val <EOL> info [ i ] [ j ] [ "<STR_LIT>" ] = self . D [ i ] [ j ] [ "<STR_LIT>" ] <EOL> return ret , info <EOL> def classes_are_presents ( self , classes ) : <EOL> m = set ( ) <EOL> for j in classes : <EOL> for i in self . N : <EOL> if self . N [ i ] . search ( j ) != None : <EOL> m . add ( i ) <EOL> return m <EOL> def show ( self ) : <EOL> for i in self . D : <EOL> print i , "<STR_LIT::>" <EOL> for j in self . D [ i ] : <EOL> print "<STR_LIT:\t>" , j , len ( self . D [ i ] [ j ] ) <EOL> for k in self . D [ i ] [ j ] : <EOL> print "<STR_LIT>" , k , len ( self . D [ i ] [ j ] [ k ] ) <EOL> def save ( self ) : <EOL> with open ( self . filename , "<STR_LIT:w>" ) as fd : <EOL> json . dump ( self . D , fd ) </s>
