deeppavlov/models/elmo/train_utils.py
def summary_gradient_updates(grads, opt, lr):
    """get summary ops for the magnitude of gradient updates"""

    # strategy:
    # make a dict of variable name -> [variable, grad, adagrad slot]
    vars_grads = {}
    for v in tf.trainable_variables():
        vars_grads[v.name] = [v, None, None]
    for g, v in grads:
        vars_grads[v.name][1] = g
        vars_grads[v.name][2] = opt.get_slot(v, 'accumulator')

    # now make summaries
    ret = []
    for vname, (v, g, a) in vars_grads.items():

        if g is None:
            continue

        if isinstance(g, tf.IndexedSlices):
            # a sparse gradient - only take norm of params that are updated
            updates = lr * g.values
            if a is not None:
                updates /= tf.sqrt(tf.gather(a, g.indices))
        else:
            updates = lr * g
            if a is not None:
                updates /= tf.sqrt(a)

        values_norm = tf.sqrt(tf.reduce_sum(v * v)) + 1.0e-7
        updates_norm = tf.sqrt(tf.reduce_sum(updates * updates))
        ret.append(tf.summary.scalar('UPDATE/' + vname.replace(":", "_"), updates_norm / values_norm))

    return ret
def summary_gradient_updates(grads, opt, lr):
    # strategy:
    # make a dict of variable name -> [variable, grad, adagrad slot]
    vars_grads = {}
    for v in tf.trainable_variables():
        vars_grads[v.name] = [v, None, None]
    for g, v in grads:
        vars_grads[v.name][1] = g
        vars_grads[v.name][2] = opt.get_slot(v, 'accumulator')

    # now make summaries
    ret = []
    for vname, (v, g, a) in vars_grads.items():

        if g is None:
            continue

        if isinstance(g, tf.IndexedSlices):
            # a sparse gradient - only take norm of params that are updated
            updates = lr * g.values
            if a is not None:
                updates /= tf.sqrt(tf.gather(a, g.indices))
        else:
            updates = lr * g
            if a is not None:
                updates /= tf.sqrt(a)

        values_norm = tf.sqrt(tf.reduce_sum(v * v)) + 1.0e-7
        updates_norm = tf.sqrt(tf.reduce_sum(updates * updates))
        ret.append(tf.summary.scalar('UPDATE/' + vname.replace(":", "_"), updates_norm / values_norm))

    return ret
def summary_gradient_updates ( grads , opt , lr ) : # strategy: # make a dict of variable name -> [variable, grad, adagrad slot] vars_grads = { } for v in tf . trainable_variables ( ) : vars_grads [ v . name ] = [ v , None , None ] for g , v in grads : vars_grads [ v . name ] [ 1 ] = g vars_grads [ v . name ] [ 2 ] = opt . get_slot ( v , 'accumulator' ) # now make summaries ret = [ ] for vname , ( v , g , a ) in vars_grads . items ( ) : if g is None : continue if isinstance ( g , tf . IndexedSlices ) : # a sparse gradient - only take norm of params that are updated updates = lr * g . values if a is not None : updates /= tf . sqrt ( tf . gather ( a , g . indices ) ) else : updates = lr * g if a is not None : updates /= tf . sqrt ( a ) values_norm = tf . sqrt ( tf . reduce_sum ( v * v ) ) + 1.0e-7 updates_norm = tf . sqrt ( tf . reduce_sum ( updates * updates ) ) ret . append ( tf . summary . scalar ( 'UPDATE/' + vname . replace ( ":" , "_" ) , updates_norm / values_norm ) ) return ret
get summary ops for the magnitude of gradient updates
get summary ops for the magnitude of gradient updates