def adagrad(loss_or_grads, params, learning_rate=1.0, epsilon=1e-06):
 grads = get_or_compute_grads(loss_or_grads, params)
 updates = OrderedDict()
 for (param, grad) in zip(params, grads):
  value = param.get_value(borrow=True)
  accu = theano.shared(np.zeros(value.shape, dtype=value.dtype), broadcastable=param.broadcastable)
  accu_new = (accu + (grad ** 2))
  updates[accu] = accu_new
  updates[param] = (param - ((learning_rate * grad) / T.sqrt((accu_new + epsilon))))
 return updates

def adagrad ( loss_or_grads , params , learning_rate = 1.0 , epsilon = 1e-06 ) : grads = get_or_compute_grads ( loss_or_grads , params ) updates = OrderedDict ( ) for ( param , grad ) in zip ( params , grads ) : value = param . get_value ( borrow = True ) accu = theano . shared ( np . zeros ( value . shape , dtype = value . dtype ) , broadcastable = param . broadcastable ) accu_new = ( accu + ( grad ** 2 ) ) updates [ accu ] = accu_new updates [ param ] = ( param - ( ( learning_rate * grad ) / T . sqrt ( ( accu_new + epsilon ) ) ) ) return updates
adagrad updates scale learning rates by dividing with the square root of accumulated squared gradients .