<s> '''<STR_LIT>''' <EOL> import xml . etree . cElementTree as ET <EOL> import os . path <EOL> import struct <EOL> import binascii <EOL> import numpy as np <EOL> import scipy . io as spio <EOL> import pyvision as pv <EOL> import pyvision . analysis . roc as roc <EOL> import gzip <EOL> BIOMETRIC_SIGNATURE = '<STR_LIT>' <EOL> PRESENTATION = '<STR_LIT>' <EOL> COMPLEX_BIOMETRIC_SIGNATURE = '<STR_LIT>' <EOL> COMPLEX_PRESENTATION = '<STR_LIT>' <EOL> COMPLEX_COMPONENT = '<STR_LIT>' <EOL> COMPLEX_DATA = '<STR_LIT>' <EOL> BEE_NONMATCH = <NUM_LIT> <EOL> BEE_MATCH = - <NUM_LIT:1> <EOL> BEE_DONTCARE = <NUM_LIT> <EOL> BEE_CODE_MAP = { <EOL> <NUM_LIT> : "<STR_LIT>" , <EOL> <NUM_LIT> : "<STR_LIT>" , <EOL> - <NUM_LIT:1> : "<STR_LIT>" , <EOL> <NUM_LIT> : "<STR_LIT>" , <EOL> } <EOL> def parseSigSet ( filename ) : <EOL> '''<STR_LIT>''' <EOL> if isinstance ( filename , str ) and filename . endswith ( '<STR_LIT>' ) : <EOL> filename = gzip . open ( filename , '<STR_LIT:rb>' ) <EOL> sigset = ET . parse ( filename ) <EOL> result = [ ] <EOL> for sig in sigset . findall ( '<STR_LIT>' ) : <EOL> name = sig . get ( '<STR_LIT:name>' ) <EOL> signature = [ ] <EOL> result . append ( ( name , signature ) ) <EOL> for pres in sig . findall ( '<STR_LIT>' ) : <EOL> presentation = { } <EOL> for key in pres . keys ( ) : <EOL> presentation [ key ] = pres . get ( key ) <EOL> signature . append ( presentation ) <EOL> for sig in sigset . findall ( BIOMETRIC_SIGNATURE ) : <EOL> name = sig . get ( '<STR_LIT:name>' ) <EOL> signature = [ ] <EOL> result . append ( ( name , signature ) ) <EOL> for pres in sig . findall ( PRESENTATION ) : <EOL> presentation = { } <EOL> for key in pres . keys ( ) : <EOL> presentation [ key ] = pres . get ( key ) <EOL> signature . append ( presentation ) <EOL> for sig in sigset . findall ( COMPLEX_BIOMETRIC_SIGNATURE ) : <EOL> name = sig . get ( '<STR_LIT:name>' ) <EOL> signature = [ ] <EOL> result . append ( ( name , signature ) ) <EOL> for pres in sig . findall ( COMPLEX_PRESENTATION ) : <EOL> presentation = { } <EOL> for key in pres . keys ( ) : <EOL> presentation [ key ] = pres . get ( key ) <EOL> for comp in pres . findall ( COMPLEX_COMPONENT ) : <EOL> for data in comp . findall ( COMPLEX_DATA ) : <EOL> for key in data . keys ( ) : <EOL> presentation [ key ] = data . get ( key ) <EOL> signature . append ( presentation ) <EOL> return result <EOL> def saveSigset ( ss , filename ) : <EOL> '''<STR_LIT>''' <EOL> if isinstance ( filename , str ) and filename . endswith ( '<STR_LIT>' ) : <EOL> filename = gzip . open ( filename , '<STR_LIT:wb>' ) <EOL> xmlss = sigset2xml ( ss ) <EOL> xmlss . write ( filename ) <EOL> def sigset2xml ( ss ) : <EOL> root = ET . Element ( "<STR_LIT>" ) <EOL> root . text = "<STR_LIT>" <EOL> for signature in ss : <EOL> sig = ET . SubElement ( root , "<STR_LIT>" ) <EOL> sig . set ( '<STR_LIT:name>' , signature [ <NUM_LIT:0> ] ) <EOL> sig . text = "<STR_LIT>" <EOL> sig . tail = "<STR_LIT>" <EOL> for presentation in signature [ <NUM_LIT:1> ] : <EOL> pres = ET . SubElement ( sig , '<STR_LIT>' ) <EOL> for key , value in presentation . iteritems ( ) : <EOL> pres . set ( key , value ) <EOL> pres . tail = "<STR_LIT>" <EOL> tree = ET . ElementTree ( root ) <EOL> return tree <EOL> def sigset2array ( ss ) : <EOL> result = [ ] <EOL> for signature in ss : <EOL> sub_id = signature [ <NUM_LIT:0> ] <EOL> if len ( signature [ <NUM_LIT:1> ] ) != <NUM_LIT:1> : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> mode = signature [ <NUM_LIT:1> ] [ <NUM_LIT:0> ] [ '<STR_LIT>' ] <EOL> file_format = signature [ <NUM_LIT:1> ] [ <NUM_LIT:0> ] [ '<STR_LIT>' ] <EOL> rec_id = signature [ <NUM_LIT:1> ] [ <NUM_LIT:0> ] [ '<STR_LIT:name>' ] <EOL> filename = signature [ <NUM_LIT:1> ] [ <NUM_LIT:0> ] [ '<STR_LIT>' ] <EOL> result . append ( [ sub_id , mode , file_format , rec_id , filename ] ) <EOL> return result <EOL> def formatSigset ( ss , n = None ) : <EOL> c = <NUM_LIT:0> <EOL> for name , data in ss : <EOL> if c == n : <EOL> break <EOL> print "<STR_LIT>" % name <EOL> for i in range ( len ( data ) ) : <EOL> print "<STR_LIT>" % i <EOL> pres = data [ i ] <EOL> for key , value in pres . iteritems ( ) : <EOL> print "<STR_LIT>" % ( key , value ) <EOL> c += <NUM_LIT:1> <EOL> def fastROC ( sorted_positives , sorted_negatives ) : <EOL> '''<STR_LIT:U+0020>''' <EOL> positives = sorted_positives <EOL> negatives = sorted_negatives <EOL> n_pos = len ( positives ) <EOL> n_neg = len ( negatives ) <EOL> assert len ( positives ) < len ( negatives ) <EOL> indexes = np . searchsorted ( negatives , positives ) <EOL> tp = ( <NUM_LIT:1.0> / n_pos ) * np . arange ( n_pos ) <EOL> fn = ( <NUM_LIT:1.0> / n_neg ) * indexes <EOL> curve = np . array ( [ tp , fn ] ) . transpose ( ) <EOL> return curve <EOL> class BEEDistanceMatrix : <EOL> def __init__ ( self , * args , ** kwargs ) : <EOL> '''<STR_LIT>''' <EOL> if isinstance ( args [ <NUM_LIT:0> ] , str ) : <EOL> self . loadFile ( * args , ** kwargs ) <EOL> elif isinstance ( args [ <NUM_LIT:0> ] , np . ndarray ) : <EOL> self . loadMatrix ( * args , ** kwargs ) <EOL> else : <EOL> raise TypeError ( "<STR_LIT>" % type ( args [ <NUM_LIT:0> ] ) ) <EOL> def loadFile ( self , filename , sigset_dir = None ) : <EOL> '''<STR_LIT>''' <EOL> self . filename = filename <EOL> self . shortname = os . path . basename ( filename ) <EOL> f = open ( filename , '<STR_LIT:rb>' ) <EOL> line = f . readline ( ) <EOL> if len ( line ) != <NUM_LIT:3> or line [ - <NUM_LIT:1> ] != "<STR_LIT>" : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> line = line . strip ( ) <EOL> if line not in [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] : <EOL> raise ValueError ( '<STR_LIT>' % line ) <EOL> self . is_distance = True <EOL> if line [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] == '<STR_LIT:S>' : <EOL> self . is_distance = False <EOL> line = f . readline ( ) . split ( ) <EOL> self . target_filename = os . path . basename ( line [ <NUM_LIT:0> ] ) <EOL> line = f . readline ( ) . split ( ) <EOL> self . query_filename = os . path . basename ( line [ <NUM_LIT:0> ] ) <EOL> line = f . readline ( ) . split ( ) <EOL> assert line [ <NUM_LIT:0> ] in [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> file_type = line [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] <EOL> self . n_queries = int ( line [ <NUM_LIT:1> ] ) <EOL> self . n_targets = int ( line [ <NUM_LIT:2> ] ) <EOL> big_endian = struct . pack ( "<STR_LIT>" , <NUM_LIT> ) <EOL> little_endian = struct . pack ( "<STR_LIT>" , <NUM_LIT> ) <EOL> if line [ <NUM_LIT:3> ] != big_endian and line [ <NUM_LIT:3> ] != little_endian : <EOL> print "<STR_LIT>" % binascii . hexlify ( line [ <NUM_LIT:3> ] ) <EOL> self . magic_number = struct . unpack_from ( "<STR_LIT>" , line [ <NUM_LIT:3> ] ) [ <NUM_LIT:0> ] <EOL> if self . magic_number == <NUM_LIT> : <EOL> byteswap = False <EOL> elif self . magic_number == <NUM_LIT> : <EOL> byteswap = True <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> if file_type == '<STR_LIT:F>' : <EOL> self . matrix = np . fromfile ( f , dtype = np . float32 ) <EOL> elif file_type == '<STR_LIT:B>' : <EOL> self . matrix = np . fromfile ( f , dtype = np . byte ) <EOL> else : <EOL> raise TypeError ( "<STR_LIT>" % file_type ) <EOL> if file_type == '<STR_LIT:F>' and byteswap : <EOL> self . matrix = self . matrix . byteswap ( ) <EOL> assert self . matrix . shape [ <NUM_LIT:0> ] == self . n_targets * self . n_queries <EOL> self . matrix = self . matrix . reshape ( self . n_queries , self . n_targets ) <EOL> if sigset_dir == None : <EOL> sigset_dir = os . path . dirname ( self . filename ) <EOL> self . queries = None <EOL> try : <EOL> ss_name = os . path . join ( sigset_dir , self . query_filename ) <EOL> self . queries = parseSigSet ( ss_name ) <EOL> assert len ( self . queries ) == self . n_queries <EOL> except : <EOL> pass <EOL> self . targets = None <EOL> try : <EOL> ss_name = os . path . join ( sigset_dir , self . target_filename ) <EOL> self . targets = parseSigSet ( ss_name ) <EOL> assert len ( self . targets ) == self . n_targets <EOL> except : <EOL> pass <EOL> def loadMatrix ( self , mat , query_filename , target_filename , sigset_dir = None , is_distance = True ) : <EOL> '''<STR_LIT>''' <EOL> self . shortname = None <EOL> if mat . dtype != np . byte : <EOL> mat = mat . astype ( np . float32 ) <EOL> self . is_distance = is_distance <EOL> self . target_filename = target_filename <EOL> self . query_filename = query_filename <EOL> self . n_queries = mat . shape [ <NUM_LIT:0> ] <EOL> self . n_targets = mat . shape [ <NUM_LIT:1> ] <EOL> self . magic_number = <NUM_LIT> <EOL> self . matrix = mat <EOL> self . queries = None <EOL> self . targets = None <EOL> if sigset_dir != None : <EOL> try : <EOL> ss_name = os . path . join ( sigset_dir , self . query_filename ) <EOL> self . queries = parseSigSet ( ss_name ) <EOL> assert len ( self . queries ) == self . n_queries <EOL> except : <EOL> print "<STR_LIT>" <EOL> print "<STR_LIT>" , ss_name <EOL> print "<STR_LIT>" , self . n_queries , "<STR_LIT>" , len ( self . queries ) <EOL> try : <EOL> ss_name = os . path . join ( sigset_dir , self . target_filename ) <EOL> self . targets = parseSigSet ( ss_name ) <EOL> assert len ( self . targets ) == self . n_targets <EOL> except : <EOL> print "<STR_LIT>" <EOL> print "<STR_LIT>" , ss_name <EOL> print "<STR_LIT>" , self . n_targets , "<STR_LIT>" , len ( self . targets ) <EOL> def cohort_norm ( self ) : <EOL> for i in range ( self . matrix . shape [ <NUM_LIT:0> ] ) : <EOL> a = self . matrix [ i , : ] <EOL> mn = a . mean ( ) <EOL> sd = a . std ( ) <EOL> self . matrix [ i , : ] = ( self . matrix [ i , : ] - mn ) / sd <EOL> def getMatchScores ( self , mask = None ) : <EOL> matches = [ ] <EOL> if self . queries != None and self . targets != None : <EOL> queries = np . array ( [ name for name , _ in self . queries ] ) <EOL> targets = np . array ( [ name for name , _ in self . targets ] ) <EOL> for i in range ( self . matrix . shape [ <NUM_LIT:0> ] ) : <EOL> if mask != None : <EOL> matches . append ( self . matrix [ i , mask . matrix [ i , : ] == BEE_MATCH ] ) <EOL> else : <EOL> query = queries [ i ] <EOL> matches . append ( self . matrix [ i , query == targets ] ) <EOL> total = <NUM_LIT:0> <EOL> for each in matches : <EOL> total += len ( each ) <EOL> scores = np . zeros ( shape = ( total ) , dtype = np . float32 ) <EOL> i = <NUM_LIT:0> <EOL> for each in matches : <EOL> s = len ( each ) <EOL> scores [ i : i + s ] = each <EOL> i += s <EOL> return scores <EOL> def getMatchScoresBySubject ( self , mask = None ) : <EOL> assert self . queries != None <EOL> assert self . targets != None <EOL> matches = { } <EOL> queries = np . array ( [ name for name , _ in self . queries ] ) <EOL> targets = np . array ( [ name for name , _ in self . targets ] ) <EOL> qnames = set ( queries ) <EOL> for name in qnames : <EOL> rows = np . nonzero ( name == queries ) [ <NUM_LIT:0> ] <EOL> cols = np . nonzero ( name == targets ) [ <NUM_LIT:0> ] <EOL> tmp = self . matrix [ rows ] [ : , cols ] <EOL> if mask != None : <EOL> m = mask . matrix [ rows ] [ : , cols ] == BEE_MATCH <EOL> matches [ name ] = tmp . flatten ( ) [ m . flatten ( ) ] <EOL> else : <EOL> matches [ name ] = tmp . flatten ( ) <EOL> if len ( matches [ name ] ) == <NUM_LIT:0> : <EOL> del matches [ name ] <EOL> return matches <EOL> def getNonMatchScores ( self , mask = None ) : <EOL> matches = [ ] <EOL> if self . queries != None and self . targets != None : <EOL> queries = np . array ( [ name for name , _ in self . queries ] ) <EOL> targets = np . array ( [ name for name , _ in self . targets ] ) <EOL> for i in range ( self . matrix . shape [ <NUM_LIT:0> ] ) : <EOL> if mask != None : <EOL> matches . append ( self . matrix [ i , mask . matrix [ i , : ] == BEE_NONMATCH ] ) <EOL> else : <EOL> query = queries [ i ] <EOL> matches . append ( self . matrix [ i , query != targets ] ) <EOL> total = <NUM_LIT:0> <EOL> for each in matches : <EOL> total += len ( each ) <EOL> scores = np . zeros ( shape = ( total ) , dtype = np . float32 ) <EOL> i = <NUM_LIT:0> <EOL> for each in matches : <EOL> s = len ( each ) <EOL> scores [ i : i + s ] = each <EOL> i += s <EOL> return scores <EOL> def asFlatArray ( self , mask = None ) : <EOL> '''<STR_LIT>''' <EOL> r , c = self . matrix . shape <EOL> result = np . zeros ( ( r * c , <NUM_LIT:4> ) , dtype = np . object ) <EOL> for i in range ( r ) : <EOL> for j in range ( c ) : <EOL> result [ c * i + j , <NUM_LIT:0> ] = i <EOL> result [ c * i + j , <NUM_LIT:1> ] = j <EOL> result [ c * i + j , <NUM_LIT:2> ] = self . matrix [ i , j ] <EOL> if BEE_CODE_MAP . has_key ( mask [ i , j ] ) : <EOL> result [ c * i + j , <NUM_LIT:3> ] = BEE_CODE_MAP [ mask [ i , j ] ] <EOL> else : <EOL> result [ c * i + j , <NUM_LIT:3> ] = "<STR_LIT>" % mask [ i , j ] <EOL> return result <EOL> def printInfo ( self ) : <EOL> print "<STR_LIT>" , self . filename <EOL> print "<STR_LIT>" , self . is_distance <EOL> print "<STR_LIT>" , self . target_filename <EOL> print "<STR_LIT>" , self . query_filename <EOL> print "<STR_LIT>" , self . n_queries <EOL> print "<STR_LIT>" , self . n_targets <EOL> print "<STR_LIT>" , self . n_targets * self . n_queries <EOL> print "<STR_LIT>" % self . magic_number <EOL> print "<STR_LIT>" , self . matrix . shape <EOL> def write ( self , filename ) : <EOL> self . save ( filename ) <EOL> def save ( self , filename ) : <EOL> '''<STR_LIT>''' <EOL> if filename . endswith ( '<STR_LIT>' ) : <EOL> self . saveBeeFormat ( filename ) <EOL> elif filename . endswith ( '<STR_LIT>' ) : <EOL> if self . is_distance : <EOL> matrix_name = '<STR_LIT>' <EOL> else : <EOL> matrix_name = '<STR_LIT>' <EOL> spio . savemat ( filename , { matrix_name : self . matrix } ) <EOL> else : <EOL> return NotImplementedError ( "<STR_LIT>" % filename ) <EOL> def saveBeeFormat ( self , filename ) : <EOL> f = open ( filename , "<STR_LIT:wb>" ) <EOL> file_type = '<STR_LIT:D>' <EOL> if self . matrix . dtype == np . byte : <EOL> file_type = '<STR_LIT:M>' <EOL> elif self . is_distance : <EOL> file_type = '<STR_LIT:D>' <EOL> else : <EOL> file_type = '<STR_LIT:S>' <EOL> f . write ( file_type ) <EOL> f . write ( "<STR_LIT>" ) <EOL> f . write ( self . target_filename + "<STR_LIT>" ) <EOL> f . write ( self . query_filename + "<STR_LIT>" ) <EOL> magic_number = struct . pack ( '<STR_LIT>' , <NUM_LIT> ) <EOL> assert len ( magic_number ) == <NUM_LIT:4> <EOL> if file_type == '<STR_LIT:M>' : <EOL> f . write ( "<STR_LIT>" % ( self . n_queries , self . n_targets , magic_number ) ) <EOL> else : <EOL> f . write ( "<STR_LIT>" % ( self . n_queries , self . n_targets , magic_number ) ) <EOL> f . write ( self . matrix ) <EOL> f . close ( ) <EOL> def histogram ( self , value_range = None , bins = <NUM_LIT:100> , normed = True , mask = None ) : <EOL> match_scores = self . getMatchScores ( mask = mask ) <EOL> nonmatch_scores = self . getNonMatchScores ( mask = mask ) <EOL> if value_range == None : <EOL> value_range = ( self . matrix . min ( ) , self . matrix . max ( ) ) <EOL> match_counts , _ = np . histogram ( match_scores , range = value_range , bins = bins , normed = normed ) <EOL> nonmatch_counts , vals = np . histogram ( nonmatch_scores , range = value_range , bins = bins , normed = normed ) <EOL> hist = pv . Table ( ) <EOL> for i in range ( len ( match_counts ) ) : <EOL> hist [ i , '<STR_LIT>' ] = vals [ i ] <EOL> hist [ i , '<STR_LIT>' ] = <NUM_LIT:0.5> * ( vals [ i ] + vals [ i + <NUM_LIT:1> ] ) <EOL> hist [ i , '<STR_LIT>' ] = vals [ i + <NUM_LIT:1> ] <EOL> hist [ i , '<STR_LIT>' ] = match_counts [ i ] <EOL> hist [ i , '<STR_LIT>' ] = nonmatch_counts [ i ] <EOL> return hist <EOL> def getROC ( self , mask = None ) : <EOL> nonmatch = self . getNonMatchScores ( mask = mask ) <EOL> match = self . getMatchScores ( mask = mask ) <EOL> return roc . ROC ( match , nonmatch , is_distance = self . is_distance ) <EOL> def getRank1 ( self , mask = None ) : <EOL> rows , _ = self . matrix . shape <EOL> queries = np . array ( [ name for name , _ in self . queries ] ) <EOL> targets = np . array ( [ name for name , _ in self . targets ] ) <EOL> success = <NUM_LIT:0.0> <EOL> count = <NUM_LIT:0.0> <EOL> for i in range ( rows ) : <EOL> row = self . matrix [ i ] <EOL> if self . is_distance : <EOL> j = row . argmin ( ) <EOL> else : <EOL> j = row . argmax ( ) <EOL> if queries [ i ] == targets [ j ] : <EOL> success += <NUM_LIT:1> <EOL> count += <NUM_LIT:1> <EOL> return success / count <EOL> def stats ( self ) : <EOL> table = pv . Table ( ) <EOL> table [ '<STR_LIT>' , '<STR_LIT>' ] = self . matrix . mean ( ) <EOL> table [ '<STR_LIT>' , '<STR_LIT>' ] = self . matrix . min ( ) <EOL> table [ '<STR_LIT>' , '<STR_LIT>' ] = self . matrix . max ( ) <EOL> return table <EOL> def __str__ ( self ) : <EOL> '''<STR_LIT>''' <EOL> file_type = { True : "<STR_LIT>" , False : "<STR_LIT>" } [ self . is_distance ] <EOL> return "<STR_LIT>" % ( self . shortname , file_type ) <EOL> def __getitem__ ( self , index ) : <EOL> '''<STR_LIT>''' <EOL> return self . matrix . __getitem__ ( index ) <EOL> def shape ( self ) : <EOL> '''<STR_LIT>''' <EOL> return self . matrix . shape <EOL> def computeMaskMatrix ( target_sigset , query_sigset , target_filename , query_filename , symmetric = True ) : <EOL> '''<STR_LIT>''' <EOL> assert len ( target_sigset ) > <NUM_LIT:0> <EOL> assert len ( query_sigset ) > <NUM_LIT:0> <EOL> target_subid = np . array ( [ each [ <NUM_LIT:0> ] for each in target_sigset ] ) <EOL> query_subid = np . array ( [ each [ <NUM_LIT:0> ] for each in query_sigset ] ) <EOL> target_recid = np . array ( [ each [ <NUM_LIT:1> ] [ <NUM_LIT:0> ] [ '<STR_LIT:name>' ] for each in target_sigset ] ) <EOL> query_recid = np . array ( [ each [ <NUM_LIT:1> ] [ <NUM_LIT:0> ] [ '<STR_LIT:name>' ] for each in query_sigset ] ) <EOL> cols = target_subid . shape [ <NUM_LIT:0> ] <EOL> rows = query_subid . shape [ <NUM_LIT:0> ] <EOL> target_subid . shape = ( <NUM_LIT:1> , cols ) <EOL> query_subid . shape = ( rows , <NUM_LIT:1> ) <EOL> target_recid . shape = ( <NUM_LIT:1> , cols ) <EOL> query_recid . shape = ( rows , <NUM_LIT:1> ) <EOL> mat = np . zeros ( ( rows , cols ) , dtype = np . byte ) <EOL> mat [ : , : ] = pv . BEE_NONMATCH <EOL> matches = target_subid == query_subid <EOL> mat [ matches ] = pv . BEE_MATCH <EOL> duplicates = target_recid == query_recid <EOL> mat [ duplicates ] = pv . BEE_DONTCARE <EOL> if symmetric and rows == cols : <EOL> ts = target_recid . flatten ( ) <EOL> qs = query_recid . flatten ( ) <EOL> if ( ts == qs ) . sum ( ) == rows : <EOL> r = np . arange ( rows ) <EOL> c = np . arange ( cols ) <EOL> r . shape = ( rows , <NUM_LIT:1> ) <EOL> c . shape = ( <NUM_LIT:1> , cols ) <EOL> tmp = r > c <EOL> mat [ tmp ] = pv . BEE_DONTCARE <EOL> return pv . BEEDistanceMatrix ( mat , query_filename , target_filename ) </s>
