<s> import sys , itertools , time , os , random <EOL> PATH_INSTALL = "<STR_LIT>" <EOL> sys . path . append ( PATH_INSTALL ) <EOL> from elsim . similarity . similarity import * <EOL> TESTS_RANDOM_SIGN = [ "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> ] <EOL> TESTS_CLOSED_SIGN = [ <EOL> [ "<STR_LIT>" , "<STR_LIT>" ] , <EOL> [ "<STR_LIT>" , "<STR_LIT>" ] , <EOL> ] <EOL> TESTS_DIFFERENT_SIGN = [ <EOL> [ "<STR_LIT>" , "<STR_LIT>" ] , <EOL> ] <EOL> TESTS_SMALL_SIGN = [ <EOL> [ "<STR_LIT>" , "<STR_LIT>" ] , <EOL> [ "<STR_LIT>" , "<STR_LIT>" ] , <EOL> [ "<STR_LIT>" , "<STR_LIT>" ] , <EOL> [ "<STR_LIT>" , "<STR_LIT>" ] , <EOL> [ "<STR_LIT>" , "<STR_LIT>" ] , <EOL> [ "<STR_LIT>" , "<STR_LIT>" ] , <EOL> ] <EOL> CONVERT_RESULT_TEST = { "<STR_LIT>" : <NUM_LIT:1> , <EOL> "<STR_LIT>" : <NUM_LIT:0> , <EOL> } <EOL> DEBUG = <NUM_LIT:0> <EOL> def test ( got , expected , fcmp ) : <EOL> if fcmp ( got , expected ) : <EOL> prefix = '<STR_LIT>' <EOL> else : <EOL> prefix = '<STR_LIT>' <EOL> if DEBUG : <EOL> print '<STR_LIT>' % ( prefix , repr ( got ) , repr ( expected ) ) <EOL> return CONVERT_RESULT_TEST [ prefix ] <EOL> def test_Idempotency ( n , x ) : <EOL> s1 = n . compress ( x + x ) <EOL> s2 = n . compress ( x ) <EOL> return test ( s1 , s2 , lambda x , y : x == y ) , s1 , s2 <EOL> def test_Monotonicity ( n , x , y ) : <EOL> s1 = n . compress ( x ) <EOL> s2 = n . compress ( x + y ) <EOL> return test ( s1 , s2 , lambda x , y : x <= y ) , s1 , s2 <EOL> def test_Symetry ( n , x , y ) : <EOL> s1 = n . compress ( x + y ) <EOL> s2 = n . compress ( y + x ) <EOL> return test ( s1 , s2 , lambda x , y : x == y ) , s1 , s2 <EOL> def test_Distributivity ( n , x , y , z ) : <EOL> s1 = n . compress ( x + y ) + n . compress ( z ) <EOL> s2 = n . compress ( x + z ) + n . compress ( y + z ) <EOL> return test ( s1 , s2 , lambda x , y : x <= y ) , s1 , s2 <EOL> def print_timing ( func ) : <EOL> def wrapper ( * arg ) : <EOL> t1 = time . time ( ) <EOL> res = func ( * arg ) <EOL> t2 = time . time ( ) <EOL> print '<STR_LIT>' % ( ( t2 - t1 ) ) <EOL> return res <EOL> return wrapper <EOL> @ print_timing <EOL> def Idempotency ( n , TESTS_TEXT ) : <EOL> print "<STR_LIT>" , <EOL> j = <NUM_LIT:0> <EOL> res = <NUM_LIT:0> <EOL> cc = <NUM_LIT:0> <EOL> for i in itertools . permutations ( TESTS_TEXT , <NUM_LIT:1> ) : <EOL> r , c1 , c2 = test_Idempotency ( n , i [ <NUM_LIT:0> ] ) <EOL> cc += c1 <EOL> cc += c2 <EOL> res += r <EOL> j += <NUM_LIT:1> <EOL> print res , "<STR_LIT:/>" , j , cc , <EOL> @ print_timing <EOL> def Monotonicity ( n , TESTS_TEXT ) : <EOL> print "<STR_LIT>" , <EOL> j = <NUM_LIT:0> <EOL> res = <NUM_LIT:0> <EOL> cc = <NUM_LIT:0> <EOL> for i in itertools . permutations ( TESTS_TEXT , <NUM_LIT:2> ) : <EOL> r , c1 , c2 = test_Monotonicity ( n , i [ <NUM_LIT:0> ] , i [ <NUM_LIT:1> ] ) <EOL> cc += c1 <EOL> cc += c2 <EOL> res += r <EOL> j += <NUM_LIT:1> <EOL> print res , "<STR_LIT:/>" , j , cc , <EOL> @ print_timing <EOL> def Symetry ( n , TESTS_TEXT ) : <EOL> print "<STR_LIT>" , <EOL> j = <NUM_LIT:0> <EOL> res = <NUM_LIT:0> <EOL> cc = <NUM_LIT:0> <EOL> for i in itertools . permutations ( TESTS_TEXT , <NUM_LIT:2> ) : <EOL> r , c1 , c2 = test_Symetry ( n , i [ <NUM_LIT:0> ] , i [ <NUM_LIT:1> ] ) <EOL> cc += c1 <EOL> cc += c2 <EOL> res += r <EOL> j += <NUM_LIT:1> <EOL> print res , "<STR_LIT:/>" , j , cc , <EOL> @ print_timing <EOL> def Distributivity ( n , TESTS_TEXT ) : <EOL> print "<STR_LIT>" , <EOL> j = <NUM_LIT:0> <EOL> cc = <NUM_LIT:0> <EOL> res = <NUM_LIT:0> <EOL> for i in itertools . permutations ( TESTS_TEXT , <NUM_LIT:3> ) : <EOL> r , c1 , c2 = test_Distributivity ( n , i [ <NUM_LIT:0> ] , i [ <NUM_LIT:1> ] , i [ <NUM_LIT:2> ] ) <EOL> cc += c1 <EOL> cc += c2 <EOL> res += r <EOL> j += <NUM_LIT:1> <EOL> print res , "<STR_LIT:/>" , j , cc , <EOL> def TestNCDPermutations ( n , ref , threshold ) : <EOL> tres , nb , idx , t = benchmark ( n . ncd , ref , threshold , lambda x , y : x <= y ) <EOL> print "<STR_LIT>" % ( tres , threshold , t , nb , idx ) <EOL> def TestNCSPermutations ( n , ref , threshold ) : <EOL> tres , nb , idx , t = benchmark ( n . ncs , ref , threshold , lambda x , y : x >= y ) <EOL> print "<STR_LIT>" % ( tres , threshold , t , nb , idx ) <EOL> def TestCMIDPermutations ( n , ref , threshold ) : <EOL> tres , nb , idx , t = benchmark ( n . cmid , ref , threshold , lambda x , y : x >= y ) <EOL> print "<STR_LIT>" % ( tres , threshold , t , nb , idx ) <EOL> def TestNCD ( n , tests , type_test ) : <EOL> TestSim ( "<STR_LIT>" , tests , type_test , n . ncd ) <EOL> def TestNCS ( n , tests , type_test ) : <EOL> TestSim ( "<STR_LIT>" , tests , type_test , n . ncs ) <EOL> def TestCMID ( n , tests , type_test ) : <EOL> TestSim ( "<STR_LIT>" , tests , type_test , n . cmid ) <EOL> def TestCMID2 ( n ) : <EOL> x = "<STR_LIT>" <EOL> y = "<STR_LIT>" <EOL> print n . cmid ( x , y ) <EOL> def TestSim ( type_sim , tests , type_test , func ) : <EOL> print type_sim , type_test <EOL> nb = <NUM_LIT:0> <EOL> print "<STR_LIT:\t>" , <EOL> t1 = time . clock ( ) <EOL> for i in tests : <EOL> val , _ = func ( i [ <NUM_LIT:0> ] , i [ <NUM_LIT:1> ] ) <EOL> print "<STR_LIT>" % ( nb , val ) , <EOL> nb += <NUM_LIT:1> <EOL> t2 = time . clock ( ) <EOL> print "<STR_LIT>" % ( t2 - t1 ) <EOL> def benchmark ( func , ref , threshold , fcmp ) : <EOL> nb = <NUM_LIT:0> <EOL> idx = <NUM_LIT:0> <EOL> tres = <NUM_LIT:0.0> <EOL> t1 = time . clock ( ) <EOL> for i in itertools . permutations ( ref ) : <EOL> perm = '<STR_LIT>' . join ( j for j in i ) <EOL> res = func ( ref , perm ) <EOL> tres += res <EOL> if fcmp ( res , threshold ) : <EOL> nb += <NUM_LIT:1> <EOL> idx += <NUM_LIT:1> <EOL> t2 = time . clock ( ) <EOL> return tres / idx , nb , idx , t2 - t1 <EOL> import math <EOL> def entropy ( data ) : <EOL> entropy = <NUM_LIT:0> <EOL> if len ( data ) == <NUM_LIT:0> : <EOL> return entropy <EOL> for x in range ( <NUM_LIT> ) : <EOL> p_x = float ( data . count ( chr ( x ) ) ) / len ( data ) <EOL> if p_x > <NUM_LIT:0> : <EOL> entropy += - p_x * math . log ( p_x , <NUM_LIT:2> ) <EOL> return entropy <EOL> def TestEntropy ( n , tests , diff ) : <EOL> nb = <NUM_LIT:0> <EOL> t1 = time . clock ( ) <EOL> for i in tests : <EOL> print n . entropy ( i [ <NUM_LIT:0> ] ) [ <NUM_LIT:0> ] , entropy ( i [ <NUM_LIT:0> ] ) <EOL> print n . entropy ( i [ <NUM_LIT:1> ] ) [ <NUM_LIT:0> ] , entropy ( i [ <NUM_LIT:1> ] ) <EOL> nb += test ( n . entropy ( i [ <NUM_LIT:0> ] ) [ <NUM_LIT:0> ] , n . entropy ( i [ <NUM_LIT:1> ] ) [ <NUM_LIT:0> ] , lambda x , y : ( max ( x , y ) - min ( x , y ) ) <= diff ) <EOL> t2 = time . clock ( ) <EOL> print "<STR_LIT>" % ( t2 - t1 , nb , len ( tests ) ) <EOL> def TestProperties ( n , data ) : <EOL> Idempotency ( n , data ) <EOL> Monotonicity ( n , data ) <EOL> Symetry ( n , data ) <EOL> Distributivity ( n , data ) <EOL> def TestSmallString ( n , data ) : <EOL> for i in data : <EOL> print i , n . ncd ( i [ <NUM_LIT:0> ] , i [ <NUM_LIT:1> ] ) <EOL> def RandomData ( ) : <EOL> l = [ ] <EOL> for i in range ( <NUM_LIT:0> , <NUM_LIT:9> ) : <EOL> l . append ( os . urandom ( random . randint ( <NUM_LIT:0> , <NUM_LIT:100> ) ) ) <EOL> return l <EOL> def _TestRDTSC ( n , m ) : <EOL> i = <NUM_LIT:0> <EOL> t0 = n . RDTSC ( ) <EOL> while i < m : <EOL> i += <NUM_LIT:1> <EOL> t1 = n . RDTSC ( ) <EOL> return t1 - t0 <EOL> def TestRDTSC ( n ) : <EOL> print _TestRDTSC ( n , <NUM_LIT:1> ) <EOL> print _TestRDTSC ( n , <NUM_LIT:10> ) <EOL> print _TestRDTSC ( n , <NUM_LIT:100> ) <EOL> print _TestRDTSC ( n , <NUM_LIT:1000> ) <EOL> def TestBenett ( n ) : <EOL> X = "<STR_LIT>" <EOL> Y = "<STR_LIT>" <EOL> n . bennett ( X ) <EOL> n . bennett ( Y ) <EOL> n . bennett ( "<STR_LIT:0>" * <NUM_LIT> ) <EOL> def TestReorg ( n ) : <EOL> X = [ "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" ] <EOL> print n . ncd ( "<STR_LIT>" . join ( j for j in X ) , "<STR_LIT>" . join ( j for j in X ) ) <EOL> for i in itertools . permutations ( X , len ( X ) ) : <EOL> print n . ncd ( "<STR_LIT>" . join ( j for j in X ) , "<STR_LIT>" . join ( j for j in i ) ) <EOL> TESTS = { "<STR_LIT>" : ZLIB_COMPRESS , <EOL> "<STR_LIT>" : BZ2_COMPRESS , <EOL> "<STR_LIT>" : LZMA_COMPRESS , <EOL> "<STR_LIT>" : XZ_COMPRESS , <EOL> "<STR_LIT>" : SNAPPY_COMPRESS , <EOL> "<STR_LIT>" : VCBLOCKSORT_COMPRESS , <EOL> } <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> n = SIMILARITY ( "<STR_LIT>" ) <EOL> TestEntropy ( n , TESTS_CLOSED_SIGN , <NUM_LIT> ) <EOL> TestEntropy ( n , TESTS_DIFFERENT_SIGN , <NUM_LIT> ) <EOL> for i in TESTS : <EOL> n . set_compress_type ( TESTS [ i ] ) <EOL> print "<STR_LIT>" , i <EOL> TestReorg ( n ) <EOL> print </s>
