128 experts, ~25B params - Train for 131072 steps on 8x8.

def xmoe2_v1_x128():
  """128 experts, ~25B params - Train for 131072 steps on 8x8."""
  hparams = xmoe2_v1()
  hparams.moe_num_experts = [16, 8]
  hparams.outer_batch_size = 8
  hparams.mesh_shape = "b0:8;b1:16"
  hparams.batch_size = 512
  hparams.learning_rate_decay_steps = 16384
  return hparams