def regularize_cost(regex, func, name='regularize_cost'):
    """
    Apply a regularizer on trainable variables matching the regex, and print
    the matched variables (only print once in multi-tower training).
    In replicated mode, it will only regularize variables within the current tower.

    If called under a TowerContext with `is_training==False`, this function returns a zero constant tensor.

    Args:
        regex (str): a regex to match variable names, e.g. "conv.*/W"
        func: the regularization function, which takes a tensor and returns a scalar tensor.
            E.g., ``tf.nn.l2_loss, tf.contrib.layers.l1_regularizer(0.001)``.

    Returns:
        tf.Tensor: a scalar, the total regularization cost.

    Example:
        .. code-block:: python

            cost = cost + regularize_cost("fc.*/W", l2_regularizer(1e-5))
    """
    assert len(regex)
    ctx = get_current_tower_context()
    if not ctx.is_training:
        # Currently cannot build the wd_cost correctly at inference,
        # because ths vs_name used in inference can be '', therefore the
        # variable filter will fail
        return tf.constant(0, dtype=tf.float32, name='empty_' + name)

    # If vars are shared, regularize all of them
    # If vars are replicated, only regularize those in the current tower
    if ctx.has_own_variables:
        params = ctx.get_collection_in_tower(tfv1.GraphKeys.TRAINABLE_VARIABLES)
    else:
        params = tfv1.trainable_variables()

    names = []

    with tfv1.name_scope(name + '_internals'):
        costs = []
        for p in params:
            para_name = p.op.name
            if re.search(regex, para_name):
                regloss = func(p)
                assert regloss.dtype.is_floating, regloss
                # Some variables may not be fp32, but it should
                # be fine to assume regularization in fp32
                if regloss.dtype != tf.float32:
                    regloss = tf.cast(regloss, tf.float32)
                costs.append(regloss)
                names.append(p.name)
        if not costs:
            return tf.constant(0, dtype=tf.float32, name='empty_' + name)

    # remove tower prefix from names, and print
    if len(ctx.vs_name):
        prefix = ctx.vs_name + '/'
        prefixlen = len(prefix)

        def f(name):
            if name.startswith(prefix):
                return name[prefixlen:]
            return name
        names = list(map(f, names))
    logger.info("regularize_cost() found {} variables to regularize.".format(len(names)))
    _log_once("The following tensors will be regularized: {}".format(', '.join(names)))

    return tf.add_n(costs, name=name)
def regularize_cost ( regex , func , name = 'regularize_cost' ) : assert len ( regex ) ctx = get_current_tower_context ( ) if not ctx . is_training : return tf . constant ( 0 , dtype = tf . float32 , name = 'empty_' + name ) if ctx . has_own_variables : params = ctx . get_collection_in_tower ( tfv1 . GraphKeys . TRAINABLE_VARIABLES ) else : params = tfv1 . trainable_variables ( ) names = [ ] with tfv1 . name_scope ( name + '_internals' ) : costs = [ ] for p in params : para_name = p . op . name if re . search ( regex , para_name ) : regloss = func ( p ) assert regloss . dtype . is_floating , regloss if regloss . dtype != tf . float32 : regloss = tf . cast ( regloss , tf . float32 ) costs . append ( regloss ) names . append ( p . name ) if not costs : return tf . constant ( 0 , dtype = tf . float32 , name = 'empty_' + name ) if len ( ctx . vs_name ) : prefix = ctx . vs_name + '/' prefixlen = len ( prefix ) def f ( name ) : if name . startswith ( prefix ) : return name [ prefixlen : ] return name names = list ( map ( f , names ) ) logger . info ( "regularize_cost() found {} variables to regularize." . format ( len ( names ) ) ) _log_once ( "The following tensors will be regularized: {}" . format ( ', ' . join ( names ) ) ) return tf . add_n ( costs , name = name )
Apply a regularizer on trainable variables matching the regex, and print
    the matched variables (only print once in multi-tower training).
    In replicated mode, it will only regularize variables within the current tower.

    If called under a TowerContext with `is_training==False`, this function returns a zero constant tensor.

    Args:
        regex (str): a regex to match variable names, e.g. "conv.*/W"
        func: the regularization function, which takes a tensor and returns a scalar tensor.
            E.g., ``tf.nn.l2_loss, tf.contrib.layers.l1_regularizer(0.001)``.

    Returns:
        tf.Tensor: a scalar, the total regularization cost.

    Example:
        .. code-block:: python

            cost = cost + regularize_cost("fc.*/W", l2_regularizer(1e-5))