def norm_constraint(tensor_var, max_norm, norm_axes=None, epsilon=1e-07):
 ndim = tensor_var.ndim
 if (norm_axes is not None):
  sum_over = tuple(norm_axes)
 elif (ndim == 2):
  sum_over = (0,)
 elif (ndim in [3, 4, 5]):
  sum_over = tuple(range(1, ndim))
 else:
  raise ValueError('Unsupported tensor dimensionality {}.Must specify `norm_axes`'.format(ndim))
 dtype = np.dtype(theano.config.floatX).type
 norms = T.sqrt(T.sum(T.sqr(tensor_var), axis=sum_over, keepdims=True))
 target_norms = T.clip(norms, 0, dtype(max_norm))
 constrained_output = (tensor_var * (target_norms / (dtype(epsilon) + norms)))
 return constrained_output

def norm_constraint ( tensor_var , max_norm , norm_axes = None , epsilon = 1e-07 ) : ndim = tensor_var . ndim if ( norm_axes is not None ) : sum_over = tuple ( norm_axes ) elif ( ndim == 2 ) : sum_over = ( 0 , ) elif ( ndim in [ 3 , 4 , 5 ] ) : sum_over = tuple ( range ( 1 , ndim ) ) else : raise ValueError ( 'Unsupported tensor dimensionality {}.Must specify `norm_axes`' . format ( ndim ) ) dtype = np . dtype ( theano . config . floatX ) . type norms = T . sqrt ( T . sum ( T . sqr ( tensor_var ) , axis = sum_over , keepdims = True ) ) target_norms = T . clip ( norms , 0 , dtype ( max_norm ) ) constrained_output = ( tensor_var * ( target_norms / ( dtype ( epsilon ) + norms ) ) ) return constrained_output
max weight norm constraints and gradient clipping this takes a tensorvariable and rescales it so that incoming weight norms are below a specified constraint value .