    def optimizer_config(self):
        """
        Return a kwarg dictionary that will be used to override optimizer
        args stored in checkpoints. This allows us to load a checkpoint and
        resume training using a different set of optimizer args, e.g., with a
        different learning rate.
        Note : Convergence issues empirically observed with fp16 on.
               Might require search for appropriate configuration.
        """
        return {
            "lr": self.args.lr[0],
            "eps": eval(self.args.adafactor_eps),
            "clip_threshold": self.args.clip_threshold,
            "decay_rate": self.args.decay_rate,
            "beta1": self.args.beta1,
            "weight_decay": self.args.weight_decay,
            "scale_parameter": self.args.scale_parameter,  # defaults to False
            "relative_step": self.args.relative_step,  # defaults to False
            "warmup_init": self.args.warmup_init,
        }
Return a kwarg dictionary that will be used to override optimizer
args stored in checkpoints. This allows us to load a checkpoint and
resume training using a different set of optimizer args, e.g., with a
different learning rate.
Note : Convergence issues empirically observed with fp16 on.
Might require search for appropriate configuration.
fairseq/optim/adafactor.py