<filename>he_j_inference/keras_model.py
#coding:utf-8
import importlib
import keras.backend as K
from keras.engine import InputSpec
from keras.layers import Input,Lambda,Dropout,Concatenate
from keras.activations import softmax
from keras.layers.core import Dense
from keras.layers import Conv2D,Average,MaxPooling2D,AveragePooling2D,Add,Flatten
from keras.layers import GlobalMaxPooling2D,GlobalAveragePooling2D,Multiply,LocallyConnected2D
from keras.models import Model
#import cv2
from keras.engine.topology import Layer
import numpy as np
import tensorflow as tf
from custom_layers import *
#from cbof import *
#from LearnToPayAttention import AttentionVGG
#batch_size=24

'''
class WildcatPool2d(Layer):
    # initialize the layer, and set an extra parameter axis. No need to include inputs parameter
    def __init__(self,kmax=0.2,kmin=0.2,alpha=0.7, **kwargs):
        #self.axis = axis
        self.kmax = kmax
        self.kmin = kmin
        self.alpha = alpha
        self.result = None
        super(WildcatPool2d, self).__init__(**kwargs)

    # first use build function to define parameters, Creates the layer weights.
    # input_shape will automatic collect input shapes to build layer
    def build(self, input_shape):
        #print(input_shape)
        super(WildcatPool2d, self).build(input_shape)

    def get_positive_k(self, k, n):
        if k <= 0:
            return 0
        elif k < 1:
            return K.cast(K.round(K.cast(n, dtype="float32")*
                   K.cast(k, dtype="float32")),dtype="int32")
        elif k > n:
            return n
        else:
            return int(k)

    # This is where the layer's logic lives. In this example, I just concat two tensors.
    def call(self, x, **kwargs):
        batch_size, h, w, num_channels = K.shape(x)[0],K.shape(x)[1],K.shape(x)[2],K.shape(x)[3]
        n = h * w  # number of regions
        kmax = self.get_positive_k(self.kmax, n)
        kmin = self.get_positive_k(self.kmin, n)
        x = K.reshape(x,(batch_size,n,num_channels))
        x = K.permute_dimensions(x,(0,2,1))
        x = tf.contrib.framework.sort(x,axis=-1,direction='DESCENDING')
        x_max = K.sum(x[:,:,:kmax],axis=-1,keepdims=False)/K.cast(kmax,dtype="float32")
        x_min = (K.sum(x[:,:,n-kmin:n],axis=-1,keepdims=False)
                     *self.alpha / K.cast(kmin,dtype="float32"))
        self.result = Average()([x_max,x_min])
        return self.result

    # return output shape
    def compute_output_shape(self, input_shape):
        #return K.int_shape(self.result)#(batch_size,num_classes)
        return tuple([input_shape[0],input_shape[3]])
'''




#################################################################

def target_category_loss(x, category_index, nb_classes):
    #batch_label=K.zeros((K.shape(x)[0],nb_classes))
    #batch_label=batch_label[:,category_index].assign(K.ones((K.shape(x)[0],)))
    batch_label=K.zeros((batch_size,nb_classes))
    batch_label=batch_label[:,category_index].assign(K.ones((batch_size,)))
    return tf.multiply(x, batch_label)

def target_category_loss_output_shape(input_shape):
    return input_shape

def normalize(x):
    # utility function to normalize a tensor by its L2 norm
    return x / (K.sqrt(K.mean(K.square(x),axis=(1,2,3),keepdims=True)) + 1e-5)


class Get_grads(Layer):
    def __init__(self, **kwargs):
        #self.axis = axis
        self.result = None
        super(Get_grads, self).__init__(**kwargs)
    def build(self, input_shape):
        print(input_shape)
        super(Get_grads, self).build(input_shape)
    def call(self, x, **kwargs):
        self.result = normalize(K.gradients(x[0], x[1])[0])
        return self.result
    def compute_output_shape(self, input_shape):
        return K.int_shape(self.result)


# 冻上base_model所有层，这样就可以正确获得bottleneck特征
def setup_to_transfer_learn(base_model):
    """Freeze all layers and compile the model"""
    for layer in base_model.layers:
        layer.trainable = False

'''
def lr_multiply(base_model):
        for layer in base_model.layers:
            layer.W_learning_rate_multiplier = args_dict.lrmult_conv
            layer.b_learning_rate_multiplier = args_dict.lrmult_conv
'''


class ModelFactory:
    """
    Model facotry for Keras default models
    """
    def __init__(self):
        self.models_ = dict(
            VGG16=dict(
                input_shape=(224, 224, 3),
                module_name="vgg16",
                last_conv_layer="block5_conv3",
            ),
            VGG19=dict(
                input_shape=(224, 224, 3),
                module_name="vgg19",
                last_conv_layer="block5_conv4",
            ),
            DenseNet121=dict(
                input_shape=(224, 224, 3),
                module_name="densenet",
                last_conv_layer="bn",
            ),
            DenseNet169=dict(
                input_shape=(224, 224, 3),
                module_name="densenet",
                last_conv_layer="bn",
            ),
            ResNet50=dict(
                input_shape=(224, 224, 3),
                module_name="resnet50",
                last_conv_layer="activation_49",
            ),
            InceptionV3=dict(
                input_shape=(299, 299, 3),
                module_name="inception_v3",
                last_conv_layer="mixed10",
            ),
            InceptionResNetV2=dict(
                input_shape=(299, 299, 3),
                module_name="inception_resnet_v2",
                last_conv_layer="conv_7b_ac",
            ),
            NASNetMobile=dict(
                input_shape=(224, 224, 3),
                module_name="nasnet",
                last_conv_layer="activation_188",
            ),
            NASNetLarge=dict(
                input_shape=(331, 331, 3),
                module_name="nasnet",
                last_conv_layer="activation_260",
            ),
            DarkNet19_448=dict(
                input_shape=(224, 224, 3),
                module_name="darknet19_448",
                last_conv_layer="activation_260",
            ),
            Xception=dict(
                input_shape=(299, 299, 3),
                module_name="xception",
                last_conv_layer="activation_260",
            ),
        )

    def get_last_conv_layer(self, model_name):
        return self.models_[model_name]["last_conv_layer"]

    def get_input_size(self, model_name):
        return self.models_[model_name]["input_shape"][:2]

    def get_model(self, class_names, model_name="DenseNet121"
                  , use_base_weights=True, weights_path=None
                  , input_shape=None, model_id=7):

        if use_base_weights is True:
            base_weights = "imagenet"
        else:
            base_weights = None

        base_model_class = getattr(
            importlib.import_module(
                #f"keras.applications.{self.models_[model_name]['module_name']}"
                 "keras.applications."+self.models_[model_name]['module_name']
            ),
            model_name)

        if input_shape is None:
            input_shape = self.models_[model_name]["input_shape"]

        img_input = Input(shape=input_shape)
        base_model = None
        base_model = base_model_class(
            include_top=False,
            input_tensor=img_input,
            input_shape=input_shape,
            weights=base_weights,
            pooling="avg")
        '''
        train bcnn with two steps:
        1.freeze base models,only train bilinear pooling and last fc layers with high lr=0.01
        2.train all layers with lr=0.001
        '''
        #setup_to_transfer_learn(base_model)

        layer_dict = dict([(layer.name, layer) for layer in base_model.layers])
        conv_outputs = None #last conv output
        if model_name=="VGG16":
            block4_conv3 = layer_dict["block4_conv3"]
            block4_conv3_outputs = block4_conv3.output
            final_conv_layer = layer_dict["block5_conv3"]
            conv_outputs = final_conv_layer.output
        if model_name=="DenseNet121" or model_name=="DenseNet169":
            final_conv_layer = layer_dict["bn"]
            conv_outputs = final_conv_layer.output
        if model_name=="InceptionV3":
            final_conv_layer = layer_dict["mixed10"]
            conv_outputs = final_conv_layer.output
        if model_id == 0:
            x = base_model.output
            '''x = conv_outputs

            ##############SE module####################
            squeeze = GlobalAveragePooling2D()(x)
            excitation = Dense(units=512 // 4, activation='relu')(squeeze)
            #excitation = Activation('relu')(excitation)
            excitation = Dense(units=512, activation='sigmoid')(excitation)
            #excitation = Activation('sigmoid')(excitation)
            excitation = Reshape((1,1,512))(excitation)
            x = Multiply()([x,excitation])
            #x = SqueezeExcitation(512)(x)
            ###########################################
            spatial_att = Conv2D(128, (1, 1), activation='relu', padding='same', name='conv6')(x)
            spatial_att = Conv2D(1, (1, 1), activation='sigmoid', padding='same', name='loc')(spatial_att)
            x = Multiply()([x,spatial_att])
            x = GlobalAveragePooling2D()(x)'''
            predictions = Dense(len(class_names), activation="softmax", name="cls_pred")(x)
        elif model_id == 1:
            loc = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp0')(conv_outputs)
            #conv6 = LocallyConnected2D(32, (3, 3), activation='relu', padding='valid', name='conv6')(cccp)
            loc = Conv2D(128, (1, 1), activation='relu', padding='same', name='conv6')(loc)
            loc = Conv2D(1, (1, 1), activation='relu', padding='same', name='loc')(loc)
            x = base_model.output
            #x = conv_outputs
            #x = x * loc
            #AttributeError: 'Tensor' object has no attribute '_keras_history'此处不能用后端函数
            #x = Multiply()([x,loc])
            #x = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp1')(x)
            #x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv7')(x)
            #x = GlobalAveragePooling2D()(x)
            #x = GlobalMaxPooling2D()(x)
            #x = Dropout(rate=0.5)(x)
            predictions = Dense(len(class_names), activation="softmax", name="cls_pred")(x)
        elif model_id == 2:
            #x = base_model.output
            x = conv_outputs
            #x = Multiply()([x,loc])
            z_l2=BilinearPooling()(x)
            #x = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp1')(x)
            #x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv7')(x)
            #x = GlobalAveragePooling2D()(x)
            #x = GlobalMaxPooling2D()(x)
            predictions = Dense(len(class_names), activation="softmax", name="cls_pred")(z_l2)
            #freeze_model = Model(inputs=img_input, output=predictions)
            #setup_to_transfer_learn(freeze_model)
            loc = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp0')(conv_outputs)
            loc = Conv2D(128, (1, 1), activation='relu', padding='same', name='conv6')(loc)
            loc = Conv2D(1, (1, 1), activation='relu', padding='same', name='loc')(loc)
        elif model_id == 3:
            loc0 = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp0')(conv_outputs)
            loc0 = Conv2D(128, (1, 1), activation='relu', padding='same', name='conv6')(loc0)
            loc0 = Conv2D(1, (1, 1), activation='relu', padding='same', name='loc0')(loc0)
            loc1 = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp1')(block4_conv3_outputs)
            loc1 = Conv2D(256, (1, 1), activation='relu', padding='same', name='conv7')(loc1)
            loc1 = Conv2D(1, (1, 1), activation='relu', padding='same', name='loc1')(loc1)
            my_resize1 = Lambda(lambda x: K.repeat_elements(x, 2, axis=1))
            x = conv_outputs
            x_att = Multiply()([x,loc0])
            loc0 = my_resize1(loc0)
            my_resize2 = Lambda(lambda x: K.repeat_elements(x, 2, axis=2))
            loc0 = my_resize2(loc0)
            #loc = Add(name='loc')([loc0, loc1])
            loc = Average(name='loc')([loc0, loc1])
            #x = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp1')(x)
            #x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv7')(x)
            #x = GlobalAveragePooling2D()(x)
            x1 = GlobalMaxPooling2D()(x_att)
            #x = Dropout(rate=0.5)(x)
            predictions = Dense(len(class_names), activation="softmax", name="cls_pred")(x1)
            x = my_resize1(x)
            x = my_resize2(x)
            x_merge = Concatenate(axis=-1)([x,block4_conv3_outputs])
            x_att1 = Multiply()([x_merge,loc])
            x2 = GlobalMaxPooling2D()(x_att1)
            predictions1 = Dense(len(class_names), activation="softmax", name="cls_pred1")(x2)
        elif model_id == 4:
            loc = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp')(conv_outputs)
            loc = Conv2D(256, (1, 1), activation='relu', padding='same', name='conv6')(loc)
            loc = Conv2D(1, (1, 1), activation='relu', padding='same', name='loc')(loc)
            #TypeError: Output tensors to a Model must be Keras tensors. Found: Tensor("Squeeze:0", shape=(?, 14, 14), dtype=float32)
            #loc = K.squeeze(loc,axis=3)
            x = conv_outputs
            x = AveragePooling2D(pool_size=(2, 2))(x)
            #x = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp2')(x)
            #x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv7')(x)
            #x = GlobalAveragePooling2D()(x)
            #x = GlobalMaxPooling2D()(x)
            x = NoisyAnd()(x)
            #x = GlobalMaxPooling2D()(x)
            #print predictions.shape
            #my_reshape = Lambda(lambda x: K.reshape(x, (-1, x.shape[3])))
            #x = my_reshape(x)
            predictions = Dense(len(class_names), activation="softmax", name="cls_pred")(x)
        elif model_id == 5:
            loc = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp0')(conv_outputs)
            loc = Conv2D(128, (1, 1), activation='relu', padding='same', name='conv6')(loc)
            loc = Conv2D(1, (1, 1), activation='relu', padding='same', name='loc')(loc)
            #x = base_model.output
            x = conv_outputs
            x = AveragePooling2D(pool_size=(2, 2))(x)
            x1 = AveragePooling2D(pool_size=(2, 2))(x)
            #x = Multiply()([x,loc])
            #x = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp1')(x)
            #x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv7')(x)
            x = Conv2D(2, (1, 1), activation='relu', padding='same', name='cccp1')(x)
            x1 = Conv2D(2, (1, 1), activation='relu', padding='same', name='cccp2')(x1)
            #x = GlobalAveragePooling2D()(x)
            #x = GlobalMaxPooling2D()(x)
            x = Softmax4D()(x)
            #x = GlobalMaxPooling2D()(x)
            x1 = Softmax4D()(x1)
            x = MaxPooling2D(pool_size=(14, 14))(x)
            x1 = MaxPooling2D(pool_size=(7, 7))(x1)
            x = Flatten(name='flatten')(x)
            x1 = Flatten(name='flatten1')(x1)
            predictions = Recalc(axis=1, name='cls_pred0')(x)
            predictions1 = Recalc(axis=1, name='cls_pred1')(x1)
            #predictions1 = Recalc(axis=1)(x1)
            predictions = Average(name='cls_pred')([predictions, predictions1])
            #predictions = Dense(len(class_names), activation="softmax", name="cls_pred")(x)
        elif model_id == 6:
            x = base_model.output
            pred = Dense(len(class_names), activation="softmax", name="pred")(x)
            target_layer = Lambda(lambda x: target_category_loss(x, 1, 2),output_shape = target_category_loss_output_shape)
            gc = target_layer(pred)
            get_loss = Lambda(lambda x: K.sum(x,axis=1))
            loss = get_loss(gc)
            grads = Get_grads()([loss, conv_outputs])
            get_weights = Lambda(lambda x: K.mean(x, axis = (1, 2),keepdims=True))
            weights = get_weights(grads)
            my_resize1 = Lambda(lambda x: K.repeat_elements(x, conv_outputs.shape[1], axis=1))
            weights = my_resize1(weights)
            my_resize2 = Lambda(lambda x: K.repeat_elements(x, conv_outputs.shape[2], axis=2))
            weights = my_resize2(weights)
            grad_cam = Multiply()([conv_outputs,weights])
            loc = Conv2D(128, (1, 1), activation='relu', padding='same', name='conv6')(conv_outputs)
            loc = Conv2D(1, (1, 1), activation='relu', padding='same', name='loc')(loc)
            #x = base_model.output
            #x = Multiply()([x,loc])
            #x = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp1')(x)
            #x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv7')(x)
            #x = GlobalAveragePooling2D()(x)
            #x = GlobalMaxPooling2D()(x)
            x = GlobalMaxPooling2D()(grad_cam)
            #x1 = MaxPooling2D(pool_size=(7, 7))(x1)
            #x = Flatten(name='flatten')(x)
            predictions = Dense(len(class_names), activation="softmax", name="cls_pred")(x)
        elif model_id == 7:
            #loc = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp0')(conv_outputs)
            #loc = Conv2D(128, (1, 1), activation='relu', padding='same', name='conv6')(loc)
            #loc = Conv2D(1, (1, 1), activation='relu', padding='same', name='loc')(loc)
            x = conv_outputs
            #x = Multiply()([x,loc])
            #num_maps=8
            classes=2
            #x = Conv2D(num_maps*classes, (1, 1), activation='relu', padding='same', name='cccp')(x)
            #x = ClassWisePool()(x)
            x = WildcatPool2d()(x)
            #x = LogSumExp()(x)
            #predictions = Recalc(axis=1, name='cls_pred')(x)#
            predictions = Dense(len(class_names), activation="softmax", name="cls_pred")(x)
            #predictions = Dense(len(class_names), activation='sigmoid', name='cls_pred')(x)
        elif model_id == 8:
            #loc = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp0')(conv_outputs)
            #loc = Conv2D(128, (1, 1), activation='relu', padding='same', name='conv6')(loc)
            #loc = Conv2D(1, (1, 1), activation='relu', padding='same', name='loc')(loc)
            x = conv_outputs
            #x = Multiply()([x,loc])
            x = LogSumExp(r=1)(x)
            predictions = Dense(len(class_names), activation="softmax", name="cls_pred")(x)
        elif model_id == 9:
            loc = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp0')(conv_outputs)
            loc = Conv2D(128, (1, 1), activation='relu', padding='same', name='conv6')(loc)
            loc = Conv2D(1, (1, 1), activation='relu', padding='same', name='loc')(loc)
            x = conv_outputs
            n_codewords=128
            x=BoF_Pooling(n_codewords, spatial_level=0)(x)
            predictions = Dense(len(class_names), activation="softmax", name="cls_pred")(x)
        elif model_id == 10:
            base_model=AttentionVGG(img_input, outputclasses=2, batchnorm=False, batchnormalizeinput=False).model
        model = Model(inputs=img_input, output=#base_model.output#predictions,#predictions1,
                                                predictions
                                                #loc

                                                )
        if weights_path == "":
            weights_path = None

        if weights_path is not None:
            #print(f"load model weights_path: {weights_path}")
            print ("load model weights_path: {}".format(weights_path))
            model.load_weights(weights_path, by_name=True)
        return model

<filename>he_j_inference/keras_model.py
#coding:utf-8
import importlib
import keras.backend as K
from keras.engine import InputSpec
from keras.layers import Input,Lambda,Dropout,Concatenate
from keras.activations import softmax
from keras.layers.core import Dense
from keras.layers import Conv2D,Average,MaxPooling2D,AveragePooling2D,Add,Flatten
from keras.layers import GlobalMaxPooling2D,GlobalAveragePooling2D,Multiply,LocallyConnected2D
from keras.models import Model
#import cv2
from keras.engine.topology import Layer
import numpy as np
import tensorflow as tf
from custom_layers import *
#from cbof import *
#from LearnToPayAttention import AttentionVGG
#batch_size=24

'''
class WildcatPool2d(Layer):
    # initialize the layer, and set an extra parameter axis. No need to include inputs parameter
    def __init__(self,kmax=0.2,kmin=0.2,alpha=0.7, **kwargs):
        #self.axis = axis
        self.kmax = kmax
        self.kmin = kmin
        self.alpha = alpha
        self.result = None
        super(WildcatPool2d, self).__init__(**kwargs)

    # first use build function to define parameters, Creates the layer weights.
    # input_shape will automatic collect input shapes to build layer
    def build(self, input_shape):
        #print(input_shape)
        super(WildcatPool2d, self).build(input_shape)

    def get_positive_k(self, k, n):
        if k <= 0:
            return 0
        elif k < 1:
            return K.cast(K.round(K.cast(n, dtype="float32")*
                   K.cast(k, dtype="float32")),dtype="int32")
        elif k > n:
            return n
        else:
            return int(k)

    # This is where the layer's logic lives. In this example, I just concat two tensors.
    def call(self, x, **kwargs):
        batch_size, h, w, num_channels = K.shape(x)[0],K.shape(x)[1],K.shape(x)[2],K.shape(x)[3]
        n = h * w  # number of regions
        kmax = self.get_positive_k(self.kmax, n)
        kmin = self.get_positive_k(self.kmin, n)
        x = K.reshape(x,(batch_size,n,num_channels))
        x = K.permute_dimensions(x,(0,2,1))
        x = tf.contrib.framework.sort(x,axis=-1,direction='DESCENDING')
        x_max = K.sum(x[:,:,:kmax],axis=-1,keepdims=False)/K.cast(kmax,dtype="float32")
        x_min = (K.sum(x[:,:,n-kmin:n],axis=-1,keepdims=False)
                     *self.alpha / K.cast(kmin,dtype="float32"))
        self.result = Average()([x_max,x_min])
        return self.result

    # return output shape
    def compute_output_shape(self, input_shape):
        #return K.int_shape(self.result)#(batch_size,num_classes)
        return tuple([input_shape[0],input_shape[3]])
'''




#################################################################

def target_category_loss(x, category_index, nb_classes):
    #batch_label=K.zeros((K.shape(x)[0],nb_classes))
    #batch_label=batch_label[:,category_index].assign(K.ones((K.shape(x)[0],)))
    batch_label=K.zeros((batch_size,nb_classes))
    batch_label=batch_label[:,category_index].assign(K.ones((batch_size,)))
    return tf.multiply(x, batch_label)

def target_category_loss_output_shape(input_shape):
    return input_shape

def normalize(x):
    # utility function to normalize a tensor by its L2 norm
    return x / (K.sqrt(K.mean(K.square(x),axis=(1,2,3),keepdims=True)) + 1e-5)


class Get_grads(Layer):
    def __init__(self, **kwargs):
        #self.axis = axis
        self.result = None
        super(Get_grads, self).__init__(**kwargs)
    def build(self, input_shape):
        print(input_shape)
        super(Get_grads, self).build(input_shape)
    def call(self, x, **kwargs):
        self.result = normalize(K.gradients(x[0], x[1])[0])
        return self.result
    def compute_output_shape(self, input_shape):
        return K.int_shape(self.result)


# 冻上base_model所有层，这样就可以正确获得bottleneck特征
def setup_to_transfer_learn(base_model):
    """Freeze all layers and compile the model"""
    for layer in base_model.layers:
        layer.trainable = False

'''
def lr_multiply(base_model):
        for layer in base_model.layers:
            layer.W_learning_rate_multiplier = args_dict.lrmult_conv
            layer.b_learning_rate_multiplier = args_dict.lrmult_conv
'''


class ModelFactory:
    """
    Model facotry for Keras default models
    """
    def __init__(self):
        self.models_ = dict(
            VGG16=dict(
                input_shape=(224, 224, 3),
                module_name="vgg16",
                last_conv_layer="block5_conv3",
            ),
            VGG19=dict(
                input_shape=(224, 224, 3),
                module_name="vgg19",
                last_conv_layer="block5_conv4",
            ),
            DenseNet121=dict(
                input_shape=(224, 224, 3),
                module_name="densenet",
                last_conv_layer="bn",
            ),
            DenseNet169=dict(
                input_shape=(224, 224, 3),
                module_name="densenet",
                last_conv_layer="bn",
            ),
            ResNet50=dict(
                input_shape=(224, 224, 3),
                module_name="resnet50",
                last_conv_layer="activation_49",
            ),
            InceptionV3=dict(
                input_shape=(299, 299, 3),
                module_name="inception_v3",
                last_conv_layer="mixed10",
            ),
            InceptionResNetV2=dict(
                input_shape=(299, 299, 3),
                module_name="inception_resnet_v2",
                last_conv_layer="conv_7b_ac",
            ),
            NASNetMobile=dict(
                input_shape=(224, 224, 3),
                module_name="nasnet",
                last_conv_layer="activation_188",
            ),
            NASNetLarge=dict(
                input_shape=(331, 331, 3),
                module_name="nasnet",
                last_conv_layer="activation_260",
            ),
            DarkNet19_448=dict(
                input_shape=(224, 224, 3),
                module_name="darknet19_448",
                last_conv_layer="activation_260",
            ),
            Xception=dict(
                input_shape=(299, 299, 3),
                module_name="xception",
                last_conv_layer="activation_260",
            ),
        )

    def get_last_conv_layer(self, model_name):
        return self.models_[model_name]["last_conv_layer"]

    def get_input_size(self, model_name):
        return self.models_[model_name]["input_shape"][:2]

    def get_model(self, class_names, model_name="DenseNet121"
                  , use_base_weights=True, weights_path=None
                  , input_shape=None, model_id=7):

        if use_base_weights is True:
            base_weights = "imagenet"
        else:
            base_weights = None

        base_model_class = getattr(
            importlib.import_module(
                #f"keras.applications.{self.models_[model_name]['module_name']}"
                 "keras.applications."+self.models_[model_name]['module_name']
            ),
            model_name)

        if input_shape is None:
            input_shape = self.models_[model_name]["input_shape"]

        img_input = Input(shape=input_shape)
        base_model = None
        base_model = base_model_class(
            include_top=False,
            input_tensor=img_input,
            input_shape=input_shape,
            weights=base_weights,
            pooling="avg")
        '''
        train bcnn with two steps:
        1.freeze base models,only train bilinear pooling and last fc layers with high lr=0.01
        2.train all layers with lr=0.001
        '''
        #setup_to_transfer_learn(base_model)

        layer_dict = dict([(layer.name, layer) for layer in base_model.layers])
        conv_outputs = None #last conv output
        if model_name=="VGG16":
            block4_conv3 = layer_dict["block4_conv3"]
            block4_conv3_outputs = block4_conv3.output
            final_conv_layer = layer_dict["block5_conv3"]
            conv_outputs = final_conv_layer.output
        if model_name=="DenseNet121" or model_name=="DenseNet169":
            final_conv_layer = layer_dict["bn"]
            conv_outputs = final_conv_layer.output
        if model_name=="InceptionV3":
            final_conv_layer = layer_dict["mixed10"]
            conv_outputs = final_conv_layer.output
        if model_id == 0:
            x = base_model.output
            '''x = conv_outputs

            ##############SE module####################
            squeeze = GlobalAveragePooling2D()(x)
            excitation = Dense(units=512 // 4, activation='relu')(squeeze)
            #excitation = Activation('relu')(excitation)
            excitation = Dense(units=512, activation='sigmoid')(excitation)
            #excitation = Activation('sigmoid')(excitation)
            excitation = Reshape((1,1,512))(excitation)
            x = Multiply()([x,excitation])
            #x = SqueezeExcitation(512)(x)
            ###########################################
            spatial_att = Conv2D(128, (1, 1), activation='relu', padding='same', name='conv6')(x)
            spatial_att = Conv2D(1, (1, 1), activation='sigmoid', padding='same', name='loc')(spatial_att)
            x = Multiply()([x,spatial_att])
            x = GlobalAveragePooling2D()(x)'''
            predictions = Dense(len(class_names), activation="softmax", name="cls_pred")(x)
        elif model_id == 1:
            loc = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp0')(conv_outputs)
            #conv6 = LocallyConnected2D(32, (3, 3), activation='relu', padding='valid', name='conv6')(cccp)
            loc = Conv2D(128, (1, 1), activation='relu', padding='same', name='conv6')(loc)
            loc = Conv2D(1, (1, 1), activation='relu', padding='same', name='loc')(loc)
            x = base_model.output
            #x = conv_outputs
            #x = x * loc
            #AttributeError: 'Tensor' object has no attribute '_keras_history'此处不能用后端函数
            #x = Multiply()([x,loc])
            #x = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp1')(x)
            #x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv7')(x)
            #x = GlobalAveragePooling2D()(x)
            #x = GlobalMaxPooling2D()(x)
            #x = Dropout(rate=0.5)(x)
            predictions = Dense(len(class_names), activation="softmax", name="cls_pred")(x)
        elif model_id == 2:
            #x = base_model.output
            x = conv_outputs
            #x = Multiply()([x,loc])
            z_l2=BilinearPooling()(x)
            #x = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp1')(x)
            #x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv7')(x)
            #x = GlobalAveragePooling2D()(x)
            #x = GlobalMaxPooling2D()(x)
            predictions = Dense(len(class_names), activation="softmax", name="cls_pred")(z_l2)
            #freeze_model = Model(inputs=img_input, output=predictions)
            #setup_to_transfer_learn(freeze_model)
            loc = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp0')(conv_outputs)
            loc = Conv2D(128, (1, 1), activation='relu', padding='same', name='conv6')(loc)
            loc = Conv2D(1, (1, 1), activation='relu', padding='same', name='loc')(loc)
        elif model_id == 3:
            loc0 = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp0')(conv_outputs)
            loc0 = Conv2D(128, (1, 1), activation='relu', padding='same', name='conv6')(loc0)
            loc0 = Conv2D(1, (1, 1), activation='relu', padding='same', name='loc0')(loc0)
            loc1 = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp1')(block4_conv3_outputs)
            loc1 = Conv2D(256, (1, 1), activation='relu', padding='same', name='conv7')(loc1)
            loc1 = Conv2D(1, (1, 1), activation='relu', padding='same', name='loc1')(loc1)
            my_resize1 = Lambda(lambda x: K.repeat_elements(x, 2, axis=1))
            x = conv_outputs
            x_att = Multiply()([x,loc0])
            loc0 = my_resize1(loc0)
            my_resize2 = Lambda(lambda x: K.repeat_elements(x, 2, axis=2))
            loc0 = my_resize2(loc0)
            #loc = Add(name='loc')([loc0, loc1])
            loc = Average(name='loc')([loc0, loc1])
            #x = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp1')(x)
            #x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv7')(x)
            #x = GlobalAveragePooling2D()(x)
            x1 = GlobalMaxPooling2D()(x_att)
            #x = Dropout(rate=0.5)(x)
            predictions = Dense(len(class_names), activation="softmax", name="cls_pred")(x1)
            x = my_resize1(x)
            x = my_resize2(x)
            x_merge = Concatenate(axis=-1)([x,block4_conv3_outputs])
            x_att1 = Multiply()([x_merge,loc])
            x2 = GlobalMaxPooling2D()(x_att1)
            predictions1 = Dense(len(class_names), activation="softmax", name="cls_pred1")(x2)
        elif model_id == 4:
            loc = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp')(conv_outputs)
            loc = Conv2D(256, (1, 1), activation='relu', padding='same', name='conv6')(loc)
            loc = Conv2D(1, (1, 1), activation='relu', padding='same', name='loc')(loc)
            #TypeError: Output tensors to a Model must be Keras tensors. Found: Tensor("Squeeze:0", shape=(?, 14, 14), dtype=float32)
            #loc = K.squeeze(loc,axis=3)
            x = conv_outputs
            x = AveragePooling2D(pool_size=(2, 2))(x)
            #x = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp2')(x)
            #x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv7')(x)
            #x = GlobalAveragePooling2D()(x)
            #x = GlobalMaxPooling2D()(x)
            x = NoisyAnd()(x)
            #x = GlobalMaxPooling2D()(x)
            #print predictions.shape
            #my_reshape = Lambda(lambda x: K.reshape(x, (-1, x.shape[3])))
            #x = my_reshape(x)
            predictions = Dense(len(class_names), activation="softmax", name="cls_pred")(x)
        elif model_id == 5:
            loc = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp0')(conv_outputs)
            loc = Conv2D(128, (1, 1), activation='relu', padding='same', name='conv6')(loc)
            loc = Conv2D(1, (1, 1), activation='relu', padding='same', name='loc')(loc)
            #x = base_model.output
            x = conv_outputs
            x = AveragePooling2D(pool_size=(2, 2))(x)
            x1 = AveragePooling2D(pool_size=(2, 2))(x)
            #x = Multiply()([x,loc])
            #x = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp1')(x)
            #x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv7')(x)
            x = Conv2D(2, (1, 1), activation='relu', padding='same', name='cccp1')(x)
            x1 = Conv2D(2, (1, 1), activation='relu', padding='same', name='cccp2')(x1)
            #x = GlobalAveragePooling2D()(x)
            #x = GlobalMaxPooling2D()(x)
            x = Softmax4D()(x)
            #x = GlobalMaxPooling2D()(x)
            x1 = Softmax4D()(x1)
            x = MaxPooling2D(pool_size=(14, 14))(x)
            x1 = MaxPooling2D(pool_size=(7, 7))(x1)
            x = Flatten(name='flatten')(x)
            x1 = Flatten(name='flatten1')(x1)
            predictions = Recalc(axis=1, name='cls_pred0')(x)
            predictions1 = Recalc(axis=1, name='cls_pred1')(x1)
            #predictions1 = Recalc(axis=1)(x1)
            predictions = Average(name='cls_pred')([predictions, predictions1])
            #predictions = Dense(len(class_names), activation="softmax", name="cls_pred")(x)
        elif model_id == 6:
            x = base_model.output
            pred = Dense(len(class_names), activation="softmax", name="pred")(x)
            target_layer = Lambda(lambda x: target_category_loss(x, 1, 2),output_shape = target_category_loss_output_shape)
            gc = target_layer(pred)
            get_loss = Lambda(lambda x: K.sum(x,axis=1))
            loss = get_loss(gc)
            grads = Get_grads()([loss, conv_outputs])
            get_weights = Lambda(lambda x: K.mean(x, axis = (1, 2),keepdims=True))
            weights = get_weights(grads)
            my_resize1 = Lambda(lambda x: K.repeat_elements(x, conv_outputs.shape[1], axis=1))
            weights = my_resize1(weights)
            my_resize2 = Lambda(lambda x: K.repeat_elements(x, conv_outputs.shape[2], axis=2))
            weights = my_resize2(weights)
            grad_cam = Multiply()([conv_outputs,weights])
            loc = Conv2D(128, (1, 1), activation='relu', padding='same', name='conv6')(conv_outputs)
            loc = Conv2D(1, (1, 1), activation='relu', padding='same', name='loc')(loc)
            #x = base_model.output
            #x = Multiply()([x,loc])
            #x = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp1')(x)
            #x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv7')(x)
            #x = GlobalAveragePooling2D()(x)
            #x = GlobalMaxPooling2D()(x)
            x = GlobalMaxPooling2D()(grad_cam)
            #x1 = MaxPooling2D(pool_size=(7, 7))(x1)
            #x = Flatten(name='flatten')(x)
            predictions = Dense(len(class_names), activation="softmax", name="cls_pred")(x)
        elif model_id == 7:
            #loc = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp0')(conv_outputs)
            #loc = Conv2D(128, (1, 1), activation='relu', padding='same', name='conv6')(loc)
            #loc = Conv2D(1, (1, 1), activation='relu', padding='same', name='loc')(loc)
            x = conv_outputs
            #x = Multiply()([x,loc])
            #num_maps=8
            classes=2
            #x = Conv2D(num_maps*classes, (1, 1), activation='relu', padding='same', name='cccp')(x)
            #x = ClassWisePool()(x)
            x = WildcatPool2d()(x)
            #x = LogSumExp()(x)
            #predictions = Recalc(axis=1, name='cls_pred')(x)#
            predictions = Dense(len(class_names), activation="softmax", name="cls_pred")(x)
            #predictions = Dense(len(class_names), activation='sigmoid', name='cls_pred')(x)
        elif model_id == 8:
            #loc = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp0')(conv_outputs)
            #loc = Conv2D(128, (1, 1), activation='relu', padding='same', name='conv6')(loc)
            #loc = Conv2D(1, (1, 1), activation='relu', padding='same', name='loc')(loc)
            x = conv_outputs
            #x = Multiply()([x,loc])
            x = LogSumExp(r=1)(x)
            predictions = Dense(len(class_names), activation="softmax", name="cls_pred")(x)
        elif model_id == 9:
            loc = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp0')(conv_outputs)
            loc = Conv2D(128, (1, 1), activation='relu', padding='same', name='conv6')(loc)
            loc = Conv2D(1, (1, 1), activation='relu', padding='same', name='loc')(loc)
            x = conv_outputs
            n_codewords=128
            x=BoF_Pooling(n_codewords, spatial_level=0)(x)
            predictions = Dense(len(class_names), activation="softmax", name="cls_pred")(x)
        elif model_id == 10:
            base_model=AttentionVGG(img_input, outputclasses=2, batchnorm=False, batchnormalizeinput=False).model
        model = Model(inputs=img_input, output=#base_model.output#predictions,#predictions1,
                                                predictions
                                                #loc

                                                )
        if weights_path == "":
            weights_path = None

        if weights_path is not None:
            #print(f"load model weights_path: {weights_path}")
            print ("load model weights_path: {}".format(weights_path))
            model.load_weights(weights_path, by_name=True)
        return model

#coding:utf-8 #import cv2 #from cbof import * #from LearnToPayAttention import AttentionVGG #batch_size=24 class WildcatPool2d(Layer):     # initialize the layer, and set an extra parameter axis. No need to include inputs parameter     def __init__(self,kmax=0.2,kmin=0.2,alpha=0.7, **kwargs):         #self.axis = axis         self.kmax = kmax         self.kmin = kmin         self.alpha = alpha         self.result = None         super(WildcatPool2d, self).__init__(**kwargs)      # first use build function to define parameters, Creates the layer weights.     # input_shape will automatic collect input shapes to build layer     def build(self, input_shape):         #print(input_shape)         super(WildcatPool2d, self).build(input_shape)      def get_positive_k(self, k, n):         if k <= 0:             return 0         elif k < 1:             return K.cast(K.round(K.cast(n, dtype="float32")*                    K.cast(k, dtype="float32")),dtype="int32")         elif k > n:             return n         else:             return int(k)      # This is where the layer's logic lives. In this example, I just concat two tensors.     def call(self, x, **kwargs):         batch_size, h, w, num_channels = K.shape(x)[0],K.shape(x)[1],K.shape(x)[2],K.shape(x)[3]         n = h * w  # number of regions         kmax = self.get_positive_k(self.kmax, n)         kmin = self.get_positive_k(self.kmin, n)         x = K.reshape(x,(batch_size,n,num_channels))         x = K.permute_dimensions(x,(0,2,1))         x = tf.contrib.framework.sort(x,axis=-1,direction='DESCENDING')         x_max = K.sum(x[:,:,:kmax],axis=-1,keepdims=False)/K.cast(kmax,dtype="float32")         x_min = (K.sum(x[:,:,n-kmin:n],axis=-1,keepdims=False)                      *self.alpha / K.cast(kmin,dtype="float32"))         self.result = Average()([x_max,x_min])         return self.result      # return output shape     def compute_output_shape(self, input_shape):         #return K.int_shape(self.result)#(batch_size,num_classes)         return tuple([input_shape[0],input_shape[3]]) ################################################################# #batch_label=K.zeros((K.shape(x)[0],nb_classes)) #batch_label=batch_label[:,category_index].assign(K.ones((K.shape(x)[0],))) # utility function to normalize a tensor by its L2 norm #self.axis = axis # 冻上base_model所有层，这样就可以正确获得bottleneck特征 Freeze all layers and compile the model def lr_multiply(base_model):         for layer in base_model.layers:             layer.W_learning_rate_multiplier = args_dict.lrmult_conv             layer.b_learning_rate_multiplier = args_dict.lrmult_conv Model facotry for Keras default models #f"keras.applications.{self.models_[model_name]['module_name']}" train bcnn with two steps:         1.freeze base models,only train bilinear pooling and last fc layers with high lr=0.01         2.train all layers with lr=0.001 #setup_to_transfer_learn(base_model) #last conv output x = conv_outputs              ##############SE module####################             squeeze = GlobalAveragePooling2D()(x)             excitation = Dense(units=512 // 4, activation='relu')(squeeze)             #excitation = Activation('relu')(excitation)             excitation = Dense(units=512, activation='sigmoid')(excitation)             #excitation = Activation('sigmoid')(excitation)             excitation = Reshape((1,1,512))(excitation)             x = Multiply()([x,excitation])             #x = SqueezeExcitation(512)(x)             ###########################################             spatial_att = Conv2D(128, (1, 1), activation='relu', padding='same', name='conv6')(x)             spatial_att = Conv2D(1, (1, 1), activation='sigmoid', padding='same', name='loc')(spatial_att)             x = Multiply()([x,spatial_att])             x = GlobalAveragePooling2D()(x) #conv6 = LocallyConnected2D(32, (3, 3), activation='relu', padding='valid', name='conv6')(cccp) #x = conv_outputs #x = x * loc #AttributeError: 'Tensor' object has no attribute '_keras_history'此处不能用后端函数 #x = Multiply()([x,loc]) #x = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp1')(x) #x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv7')(x) #x = GlobalAveragePooling2D()(x) #x = GlobalMaxPooling2D()(x) #x = Dropout(rate=0.5)(x) #x = base_model.output #x = Multiply()([x,loc]) #x = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp1')(x) #x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv7')(x) #x = GlobalAveragePooling2D()(x) #x = GlobalMaxPooling2D()(x) #freeze_model = Model(inputs=img_input, output=predictions) #setup_to_transfer_learn(freeze_model) #loc = Add(name='loc')([loc0, loc1]) #x = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp1')(x) #x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv7')(x) #x = GlobalAveragePooling2D()(x) #x = Dropout(rate=0.5)(x) #TypeError: Output tensors to a Model must be Keras tensors. Found: Tensor("Squeeze:0", shape=(?, 14, 14), dtype=float32) #loc = K.squeeze(loc,axis=3) #x = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp2')(x) #x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv7')(x) #x = GlobalAveragePooling2D()(x) #x = GlobalMaxPooling2D()(x) #x = GlobalMaxPooling2D()(x) #print predictions.shape #my_reshape = Lambda(lambda x: K.reshape(x, (-1, x.shape[3]))) #x = my_reshape(x) #x = base_model.output #x = Multiply()([x,loc]) #x = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp1')(x) #x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv7')(x) #x = GlobalAveragePooling2D()(x) #x = GlobalMaxPooling2D()(x) #x = GlobalMaxPooling2D()(x) #predictions1 = Recalc(axis=1)(x1) #predictions = Dense(len(class_names), activation="softmax", name="cls_pred")(x) #x = base_model.output #x = Multiply()([x,loc]) #x = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp1')(x) #x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv7')(x) #x = GlobalAveragePooling2D()(x) #x = GlobalMaxPooling2D()(x) #x1 = MaxPooling2D(pool_size=(7, 7))(x1) #x = Flatten(name='flatten')(x) #loc = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp0')(conv_outputs) #loc = Conv2D(128, (1, 1), activation='relu', padding='same', name='conv6')(loc) #loc = Conv2D(1, (1, 1), activation='relu', padding='same', name='loc')(loc) #x = Multiply()([x,loc]) #num_maps=8 #x = Conv2D(num_maps*classes, (1, 1), activation='relu', padding='same', name='cccp')(x) #x = ClassWisePool()(x) #x = LogSumExp()(x) #predictions = Recalc(axis=1, name='cls_pred')(x)# #predictions = Dense(len(class_names), activation='sigmoid', name='cls_pred')(x) #loc = Conv2D(512, (1, 1), activation='relu', padding='same', name='cccp0')(conv_outputs) #loc = Conv2D(128, (1, 1), activation='relu', padding='same', name='conv6')(loc) #loc = Conv2D(1, (1, 1), activation='relu', padding='same', name='loc')(loc) #x = Multiply()([x,loc]) #base_model.output#predictions,#predictions1, #loc #print(f"load model weights_path: {weights_path}")