<s> import numpy <EOL> import sys <EOL> import cpa . sqltools <EOL> from dbconnect import * <EOL> from properties import Properties <EOL> from datamodel import DataModel <EOL> db = DBConnect . getInstance ( ) <EOL> p = Properties . getInstance ( ) <EOL> dm = DataModel . getInstance ( ) <EOL> temp_stump_table = "<STR_LIT>" <EOL> temp_score_table = "<STR_LIT>" <EOL> temp_class_table = "<STR_LIT>" <EOL> filter_table_prefix = '<STR_LIT>' <EOL> def translate ( weaklearners ) : <EOL> '''<STR_LIT>''' <EOL> num_features = len ( weaklearners ) <EOL> nClasses = len ( weaklearners [ <NUM_LIT:0> ] [ <NUM_LIT:2> ] ) <EOL> has_classifier_function = False <EOL> if p . db_type . lower ( ) == '<STR_LIT>' : <EOL> thresholds = numpy . array ( [ wl [ <NUM_LIT:1> ] for wl in weaklearners ] ) <EOL> a = numpy . array ( [ wl [ <NUM_LIT:2> ] for wl in weaklearners ] ) <EOL> b = numpy . array ( [ wl [ <NUM_LIT:3> ] for wl in weaklearners ] ) <EOL> db . setup_sqlite_classifier ( thresholds , a , b ) <EOL> return "<STR_LIT>" % ( "<STR_LIT:U+002C>" . join ( [ wl [ <NUM_LIT:0> ] for wl in weaklearners ] ) ) <EOL> if p . db_type . lower ( ) == '<STR_LIT>' : <EOL> try : <EOL> res = db . execute ( "<STR_LIT>" ) <EOL> has_classifier_function = len ( res ) > <NUM_LIT:0> <EOL> except : <EOL> has_classifier_function = False <EOL> if has_classifier_function : <EOL> num_stumps = len ( weaklearners ) <EOL> featurenames = "<STR_LIT>" + "<STR_LIT:U+002C>" . join ( [ wl [ <NUM_LIT:0> ] for wl in weaklearners ] ) <EOL> thresholds = "<STR_LIT>" + "<STR_LIT:U+002C>" . join ( [ str ( wl [ <NUM_LIT:1> ] ) for wl in weaklearners ] ) <EOL> base = [ sum ( [ wl [ <NUM_LIT:3> ] [ k ] for wl in weaklearners ] ) for k in range ( nClasses ) ] <EOL> weights = "<STR_LIT:U+002C>" . join ( [ "<STR_LIT:U+002C>" . join ( [ str ( base [ k ] ) ] + [ str ( wl [ <NUM_LIT:2> ] [ k ] - wl [ <NUM_LIT:3> ] [ k ] ) for wl in weaklearners ] ) for k in range ( nClasses ) ] ) <EOL> return "<STR_LIT>" % ( num_stumps + <NUM_LIT:1> , featurenames , thresholds , weights ) <EOL> else : <EOL> class_scores = [ '<STR_LIT:+>' . join ( [ '<STR_LIT>' % ( feature , threshold , a [ i ] , b [ i ] ) for ( feature , threshold , a , b , ignore ) in weaklearners ] ) for i in range ( nClasses ) ] <EOL> return "<STR_LIT>" % ( "<STR_LIT:U+002C>" . join ( class_scores ) , "<STR_LIT:\n>" . join ( [ "<STR_LIT>" % ( score , idx + <NUM_LIT:1> ) for idx , score in enumerate ( class_scores ) ] ) ) <EOL> def FilterObjectsFromClassN ( clNum , weaklearners , filterKeys ) : <EOL> '''<STR_LIT>''' <EOL> class_query = translate ( weaklearners ) <EOL> if filterKeys != [ ] : <EOL> if isinstance ( filterKeys , str ) : <EOL> whereclause = filterKeys + "<STR_LIT>" <EOL> else : <EOL> isImKey = len ( filterKeys [ <NUM_LIT:0> ] ) == len ( image_key_columns ( ) ) <EOL> if isImKey : <EOL> whereclause = GetWhereClauseForImages ( filterKeys ) + "<STR_LIT>" <EOL> else : <EOL> whereclause = GetWhereClauseForObjects ( filterKeys ) + "<STR_LIT>" <EOL> else : <EOL> whereclause = "<STR_LIT>" <EOL> return db . execute ( '<STR_LIT>' + UniqueObjectClause ( ) + '<STR_LIT>' % ( p . object_table , whereclause , class_query , clNum ) ) <EOL> def object_scores ( weaklearners ) : <EOL> stump_stmnts , score_stmnts , find_max_query , _ , _ = translate ( weaklearners ) <EOL> db . execute ( '<STR_LIT>' ) <EOL> db . execute ( '<STR_LIT>' ) <EOL> [ db . execute ( stump_query ) for stump_query in stump_stmnts ] <EOL> [ db . execute ( score_query ) for score_query in score_stmnts ] <EOL> col_names = db . GetColumnNames ( '<STR_LIT>' ) <EOL> col_types = db . GetColumnTypes ( '<STR_LIT>' ) <EOL> type_mapping = { long : '<STR_LIT>' , float : '<STR_LIT>' } <EOL> dtype = numpy . dtype ( [ ( name , type_mapping [ type ] ) <EOL> for name , type in zip ( col_names , col_types ) ] ) <EOL> res = db . execute ( '<STR_LIT>' ) <EOL> return numpy . array ( map ( tuple , res ) , dtype ) <EOL> def create_perobject_class_table ( classnames , rules ) : <EOL> '''<STR_LIT>''' <EOL> nClasses = len ( classnames ) <EOL> if p . class_table is None : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> index_cols = UniqueObjectClause ( ) <EOL> class_cols = UniqueObjectClause ( ) + '<STR_LIT>' <EOL> class_col_defs = object_key_defs ( ) + '<STR_LIT>' % ( max ( [ len ( c ) for c in classnames ] ) + <NUM_LIT:1> ) + '<STR_LIT>' <EOL> db . execute ( '<STR_LIT>' % ( p . class_table ) ) <EOL> db . execute ( '<STR_LIT>' % ( p . class_table , class_col_defs ) ) <EOL> db . execute ( '<STR_LIT>' % ( p . class_table , p . class_table , index_cols ) ) <EOL> case_expr = '<STR_LIT>' % ( translate ( rules ) ) + '<STR_LIT>' . join ( [ "<STR_LIT>" % ( n + <NUM_LIT:1> , classnames [ n ] ) for n in range ( nClasses ) ] ) + "<STR_LIT>" <EOL> case_expr2 = '<STR_LIT>' % ( translate ( rules ) ) + '<STR_LIT>' . join ( [ "<STR_LIT>" % ( n + <NUM_LIT:1> , n + <NUM_LIT:1> ) for n in range ( nClasses ) ] ) + "<STR_LIT>" <EOL> db . execute ( '<STR_LIT>' % ( p . class_table , class_cols , index_cols , case_expr , case_expr2 , p . object_table ) ) <EOL> db . Commit ( ) <EOL> def _objectify ( p , field ) : <EOL> return "<STR_LIT>" % ( p . object_table , field ) <EOL> def _where_clauses ( p , dm , filter_name ) : <EOL> imkeys = dm . GetAllImageKeys ( filter_name ) <EOL> imkeys . sort ( ) <EOL> stepsize = max ( len ( imkeys ) / <NUM_LIT:100> , <NUM_LIT:50> ) <EOL> key_thresholds = imkeys [ - <NUM_LIT:1> : <NUM_LIT:1> : - stepsize ] <EOL> key_thresholds . reverse ( ) <EOL> if len ( key_thresholds ) == <NUM_LIT:0> : <EOL> return [ '<STR_LIT>' ] <EOL> if p . table_id : <EOL> def splitter ( ) : <EOL> yield "<STR_LIT>" % ( _objectify ( p , p . table_id ) , key_thresholds [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] , <EOL> _objectify ( p , p . image_id ) , key_thresholds [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] ) <EOL> for lo , hi in zip ( key_thresholds [ : - <NUM_LIT:1> ] , key_thresholds [ <NUM_LIT:1> : ] ) : <EOL> if lo [ <NUM_LIT:0> ] == hi [ <NUM_LIT:0> ] : <EOL> yield "<STR_LIT>" % ( _objectify ( p , p . table_id ) , lo [ <NUM_LIT:0> ] , <EOL> _objectify ( p , p . image_id ) , lo [ <NUM_LIT:1> ] , <EOL> _objectify ( p , p . image_id ) , hi [ <NUM_LIT:1> ] ) <EOL> else : <EOL> yield "<STR_LIT>" % ( _objectify ( p , p . table_id ) , lo [ <NUM_LIT:0> ] , <EOL> _objectify ( p , p . image_id ) , lo [ <NUM_LIT:1> ] ) <EOL> yield "<STR_LIT>" % ( _objectify ( p , p . table_id ) , hi [ <NUM_LIT:0> ] , <EOL> _objectify ( p , p . image_id ) , hi [ <NUM_LIT:1> ] ) <EOL> return list ( splitter ( ) ) <EOL> else : <EOL> return ( [ "<STR_LIT>" % ( _objectify ( p , p . image_id ) , key_thresholds [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] ) ] + <EOL> [ "<STR_LIT>" <EOL> % ( _objectify ( p , p . image_id ) , lo [ <NUM_LIT:0> ] , _objectify ( p , p . image_id ) , hi [ <NUM_LIT:0> ] ) <EOL> for lo , hi in zip ( key_thresholds [ : - <NUM_LIT:1> ] , key_thresholds [ <NUM_LIT:1> : ] ) ] ) <EOL> def PerImageCounts ( weaklearners , filter_name = None , cb = None ) : <EOL> '''<STR_LIT>''' <EOL> def do_by_steps ( class_query , tables , filter_name , result_clauses ) : <EOL> filter_clause = '<STR_LIT>' <EOL> join_clause = '<STR_LIT>' <EOL> if filter_name is not None : <EOL> filter = p . _filters [ filter_name ] <EOL> if isinstance ( filter , cpa . sqltools . OldFilter ) : <EOL> join_table = '<STR_LIT>' % str ( filter ) <EOL> else : <EOL> if p . object_table in tables : <EOL> join_table = None <EOL> else : <EOL> join_table = p . object_table <EOL> filter_clause = str ( filter ) <EOL> if join_table : <EOL> join_clause = '<STR_LIT>' % ( join_table , '<STR_LIT:U+002C>' . join ( image_key_columns ( ) ) ) <EOL> if cb : <EOL> result = [ ] <EOL> wheres = _where_clauses ( p , dm , filter_name ) <EOL> num_clauses = len ( wheres ) <EOL> for idx , where_clause in enumerate ( wheres ) : <EOL> if filter_clause is not None : <EOL> where_clause += '<STR_LIT>' + filter_clause <EOL> result += [ db . execute ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> % ( UniqueImageClause ( p . object_table ) , <EOL> class_query , result_clauses , tables , <EOL> join_clause , where_clause , <EOL> UniqueImageClause ( p . object_table ) ) , <EOL> silent = ( idx > <NUM_LIT:10> ) ) ] <EOL> cb ( min ( <NUM_LIT:1> , idx / float ( num_clauses ) ) ) <EOL> return sum ( result , [ ] ) <EOL> else : <EOL> return db . execute ( '<STR_LIT>' % <EOL> ( imkeys , class_query , result_clauses , tables , join_clause , filter_clause , imkeys ) ) <EOL> if p . area_scoring_column is None : <EOL> result_clauses = '<STR_LIT>' <EOL> else : <EOL> result_clauses = '<STR_LIT>' % ( _objectify ( p , p . area_scoring_column ) ) <EOL> class_query = translate ( weaklearners ) <EOL> results = do_by_steps ( class_query , p . object_table , filter_name , result_clauses ) <EOL> counts = { } <EOL> keylen = <NUM_LIT:1> + len ( image_key_columns ( ) ) <EOL> for r in results : <EOL> counts [ r [ : keylen ] ] = r [ keylen : ] <EOL> num_classes = len ( weaklearners [ <NUM_LIT:0> ] [ <NUM_LIT:2> ] ) <EOL> def get_count ( im_key , classnum ) : <EOL> return counts . get ( tuple ( list ( im_key ) + [ classnum ] ) , [ <NUM_LIT:0> ] ) [ <NUM_LIT:0> ] <EOL> def get_area ( im_key , classnum ) : <EOL> return counts . get ( tuple ( list ( im_key ) + [ classnum ] ) , [ <NUM_LIT:0> , <NUM_LIT:0> ] ) [ <NUM_LIT:1> ] <EOL> def get_results ( ) : <EOL> for imkey in dm . GetImageKeysAndObjectCounts ( filter_name ) : <EOL> if p . area_scoring_column is None : <EOL> yield list ( imkey [ <NUM_LIT:0> ] ) + [ get_count ( imkey [ <NUM_LIT:0> ] , cl ) for cl in range ( <NUM_LIT:1> , num_classes + <NUM_LIT:1> ) ] <EOL> else : <EOL> yield list ( imkey [ <NUM_LIT:0> ] ) + [ get_count ( imkey [ <NUM_LIT:0> ] , cl ) for cl in range ( <NUM_LIT:1> , num_classes + <NUM_LIT:1> ) ] + [ get_area ( imkey [ <NUM_LIT:0> ] , cl ) for cl in range ( <NUM_LIT:1> , num_classes + <NUM_LIT:1> ) ] <EOL> return list ( get_results ( ) ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> from trainingset import TrainingSet <EOL> from StringIO import StringIO <EOL> import fastgentleboostingmulticlass <EOL> from datatable import DataGrid <EOL> import wx <EOL> p = Properties . getInstance ( ) <EOL> db = DBConnect . getInstance ( ) <EOL> dm = DataModel . getInstance ( ) <EOL> props = '<STR_LIT>' <EOL> ts = '<STR_LIT>' <EOL> nRules = <NUM_LIT:5> <EOL> filter = '<STR_LIT>' <EOL> p . LoadFile ( props ) <EOL> trainingSet = TrainingSet ( p ) <EOL> trainingSet . Load ( ts ) <EOL> output = StringIO ( ) <EOL> print '<STR_LIT>' + str ( nRules ) + '<STR_LIT>' <EOL> weaklearners = fastgentleboostingmulticlass . train ( trainingSet . colnames , <EOL> nRules , trainingSet . label_matrix , <EOL> trainingSet . values , output ) <EOL> table = PerImageCounts ( weaklearners , filter_name = filter ) <EOL> table . sort ( ) <EOL> labels = [ '<STR_LIT>' , '<STR_LIT:image>' ] + list ( trainingSet . labels ) + list ( trainingSet . labels ) <EOL> print labels <EOL> for row in table : <EOL> print row </s>
