def __init__(
        self,
        learning_rate=0.01,
        momentum=0.9,
        clip_kl=0.01,
        kfac_update=2,
        stats_accum_iter=60,
        full_stats_init=False,
        cold_iter=100,
        cold_lr=None,
        async_=False,
        async_stats=False,
        epsilon=1e-2,
        stats_decay=0.95,
        blockdiag_bias=False,
        channel_fac=False,
        factored_damping=False,
        approxT2=False,
        use_float64=False,
        weight_decay_dict={},
        max_grad_norm=0.5,
        scope='kfac',
        summary_labels=()
    ):
        """
        Initializes a KFAC optimizer.

        For more information on arguments, see the Kfac Optimization paper https://arxiv.org/pdf/1503.05671.pdf
        """
        self.max_grad_norm = max_grad_norm
        self._lr = learning_rate
        self._momentum = momentum
        self._clip_kl = clip_kl
        self._channel_fac = channel_fac
        self._kfac_update = kfac_update
        self._async = async_
        self._async_stats = async_stats
        self._epsilon = epsilon
        self._stats_decay = stats_decay
        self._blockdiag_bias = blockdiag_bias
        self._approxT2 = approxT2
        self._use_float64 = use_float64
        self._factored_damping = factored_damping
        self._cold_iter = cold_iter
        if cold_lr == None:
            # good heuristics
            self._cold_lr = self._lr# * 3.
        else:
            self._cold_lr = cold_lr
        self._stats_accum_iter = stats_accum_iter
        self._weight_decay_dict = weight_decay_dict
        self._diag_init_coeff = 0.
        self._full_stats_init = full_stats_init
        if not self._full_stats_init:
            self._stats_accum_iter = self._cold_iter

        self.sgd_step = tf.Variable(0, name='KFAC/sgd_step', trainable=False)
        self.global_step = tf.Variable(
            0, name='KFAC/global_step', trainable=False)
        self.cold_step = tf.Variable(0, name='KFAC/cold_step', trainable=False)
        self.factor_step = tf.Variable(
            0, name='KFAC/factor_step', trainable=False)
        self.stats_step = tf.Variable(
            0, name='KFAC/stats_step', trainable=False)
        self.vFv = tf.Variable(0., name='KFAC/vFv', trainable=False)

        self.factors = {}
        self.param_vars = []
        self.stats = {}
        self.stats_eigen = {}

        super(KFAC, self).__init__(scope=scope, summary_labels=summary_labels)
def __init__ ( self , learning_rate = 0.01 , momentum = 0.9 , clip_kl = 0.01 , kfac_update = 2 , stats_accum_iter = 60 , full_stats_init = False , cold_iter = 100 , cold_lr = None , async_ = False , async_stats = False , epsilon = 1e-2 , stats_decay = 0.95 , blockdiag_bias = False , channel_fac = False , factored_damping = False , approxT2 = False , use_float64 = False , weight_decay_dict = { } , max_grad_norm = 0.5 , scope = 'kfac' , summary_labels = ( ) ) : self . max_grad_norm = max_grad_norm self . _lr = learning_rate self . _momentum = momentum self . _clip_kl = clip_kl self . _channel_fac = channel_fac self . _kfac_update = kfac_update self . _async = async_ self . _async_stats = async_stats self . _epsilon = epsilon self . _stats_decay = stats_decay self . _blockdiag_bias = blockdiag_bias self . _approxT2 = approxT2 self . _use_float64 = use_float64 self . _factored_damping = factored_damping self . _cold_iter = cold_iter if cold_lr == None : self . _cold_lr = self . _lr else : self . _cold_lr = cold_lr self . _stats_accum_iter = stats_accum_iter self . _weight_decay_dict = weight_decay_dict self . _diag_init_coeff = 0. self . _full_stats_init = full_stats_init if not self . _full_stats_init : self . _stats_accum_iter = self . _cold_iter self . sgd_step = tf . Variable ( 0 , name = 'KFAC/sgd_step' , trainable = False ) self . global_step = tf . Variable ( 0 , name = 'KFAC/global_step' , trainable = False ) self . cold_step = tf . Variable ( 0 , name = 'KFAC/cold_step' , trainable = False ) self . factor_step = tf . Variable ( 0 , name = 'KFAC/factor_step' , trainable = False ) self . stats_step = tf . Variable ( 0 , name = 'KFAC/stats_step' , trainable = False ) self . vFv = tf . Variable ( 0. , name = 'KFAC/vFv' , trainable = False ) self . factors = { } self . param_vars = [ ] self . stats = { } self . stats_eigen = { } super ( KFAC , self ) . __init__ ( scope = scope , summary_labels = summary_labels )
Initializes a KFAC optimizer.

        For more information on arguments, see the Kfac Optimization paper https://arxiv.org/pdf/1503.05671.pdf