def init_optimizer(self, kvstore='local', optimizer='sgd',
                       optimizer_params=(('learning_rate', 0.01),), force_init=False):
        """Installs and initializes optimizers.

        Parameters
        ----------
        kvstore : str or KVStore
            Default `'local'`.
        optimizer : str or Optimizer
            Default `'sgd'`
        optimizer_params : dict
            Default `(('learning_rate', 0.01),)`. The default value is not a dictionary,
            just to avoid pylint warning of dangerous default values.
        force_init : bool
            Default ``False``, indicating whether we should force re-initializing the
            optimizer in the case an optimizer is already installed.
        """
        assert self.binded and self.params_initialized

        if self.optimizer_initialized and not force_init:
            self.logger.warning('optimizer already initialized, ignoring...')
            return

        if self._params_dirty:
            self._sync_params_from_devices()

        (kvstore, update_on_kvstore) = \
                _create_kvstore(kvstore, len(self._context), self._arg_params)

        batch_size = self._exec_group.batch_size
        if kvstore and 'dist' in kvstore.type and '_sync' in kvstore.type:
            batch_size *= kvstore.num_workers
        rescale_grad = 1.0/batch_size

        idx2name = {}
        if update_on_kvstore:
            idx2name.update(enumerate(self._exec_group.param_names))
        else:
            for k in range(len(self._context)):
                idx2name.update({i*len(self._context)+k: n
                                 for i, n in enumerate(self._exec_group.param_names)})
        if isinstance(optimizer, str):
            optimizer_params = dict(optimizer_params)
            if 'rescale_grad' not in optimizer_params:
                optimizer_params['rescale_grad'] = rescale_grad
            optimizer = opt.create(optimizer,
                                   sym=self.symbol, param_idx2name=idx2name,
                                   **optimizer_params)
        else:
            assert isinstance(optimizer, opt.Optimizer)
            if optimizer.rescale_grad != rescale_grad:
                #pylint: disable=no-member
                warnings.warn(
                    "Optimizer created manually outside Module but rescale_grad " +
                    "is not normalized to 1.0/batch_size/num_workers (%s vs. %s). "%(
                        optimizer.rescale_grad, rescale_grad) +
                    "Is this intended?", stacklevel=2)
            if not optimizer.idx2name:
                optimizer.idx2name = idx2name.copy()

        self._optimizer = optimizer
        self._kvstore = kvstore
        self._update_on_kvstore = update_on_kvstore
        self._updater = None

        if kvstore:
            if self._compression_params:
                kvstore.set_gradient_compression(self._compression_params)
            if update_on_kvstore:
                kvstore.set_optimizer(self._optimizer)
            # copy initialized local parameters to kvstore
            _initialize_kvstore(kvstore=kvstore,
                                param_arrays=self._exec_group.param_arrays,
                                arg_params=self._arg_params,
                                param_names=self._param_names,
                                update_on_kvstore=update_on_kvstore)

        if not update_on_kvstore:
            self._updater = opt.get_updater(optimizer)

        self.optimizer_initialized = True

        if self._preload_opt_states is not None:
            self.load_optimizer_states(self._preload_opt_states)
            self._preload_opt_states = None
def init_optimizer ( self , kvstore = 'local' , optimizer = 'sgd' , optimizer_params = ( ( 'learning_rate' , 0.01 ) , ) , force_init = False ) : assert self . binded and self . params_initialized if self . optimizer_initialized and not force_init : self . logger . warning ( 'optimizer already initialized, ignoring...' ) return if self . _params_dirty : self . _sync_params_from_devices ( ) ( kvstore , update_on_kvstore ) = _create_kvstore ( kvstore , len ( self . _context ) , self . _arg_params ) batch_size = self . _exec_group . batch_size if kvstore and 'dist' in kvstore . type and '_sync' in kvstore . type : batch_size *= kvstore . num_workers rescale_grad = 1.0 / batch_size idx2name = { } if update_on_kvstore : idx2name . update ( enumerate ( self . _exec_group . param_names ) ) else : for k in range ( len ( self . _context ) ) : idx2name . update ( { i * len ( self . _context ) + k : n for i , n in enumerate ( self . _exec_group . param_names ) } ) if isinstance ( optimizer , str ) : optimizer_params = dict ( optimizer_params ) if 'rescale_grad' not in optimizer_params : optimizer_params [ 'rescale_grad' ] = rescale_grad optimizer = opt . create ( optimizer , sym = self . symbol , param_idx2name = idx2name , ** optimizer_params ) else : assert isinstance ( optimizer , opt . Optimizer ) if optimizer . rescale_grad != rescale_grad : warnings . warn ( "Optimizer created manually outside Module but rescale_grad " + "is not normalized to 1.0/batch_size/num_workers (%s vs. %s). " % ( optimizer . rescale_grad , rescale_grad ) + "Is this intended?" , stacklevel = 2 ) if not optimizer . idx2name : optimizer . idx2name = idx2name . copy ( ) self . _optimizer = optimizer self . _kvstore = kvstore self . _update_on_kvstore = update_on_kvstore self . _updater = None if kvstore : if self . _compression_params : kvstore . set_gradient_compression ( self . _compression_params ) if update_on_kvstore : kvstore . set_optimizer ( self . _optimizer ) _initialize_kvstore ( kvstore = kvstore , param_arrays = self . _exec_group . param_arrays , arg_params = self . _arg_params , param_names = self . _param_names , update_on_kvstore = update_on_kvstore ) if not update_on_kvstore : self . _updater = opt . get_updater ( optimizer ) self . optimizer_initialized = True if self . _preload_opt_states is not None : self . load_optimizer_states ( self . _preload_opt_states ) self . _preload_opt_states = None
Installs and initializes optimizers.

        Parameters
        ----------
        kvstore : str or KVStore
            Default `'local'`.
        optimizer : str or Optimizer
            Default `'sgd'`
        optimizer_params : dict
            Default `(('learning_rate', 0.01),)`. The default value is not a dictionary,
            just to avoid pylint warning of dangerous default values.
        force_init : bool
            Default ``False``, indicating whether we should force re-initializing the
            optimizer in the case an optimizer is already installed.