Mixture of experts (16 experts).


  623M Params, einsum=1.09e13

  Returns:
    a hparams

def xmoe_tr_1d():
  """Mixture of experts (16 experts).


  623M Params, einsum=1.09e13

  Returns:
    a hparams
  """
  hparams = xmoe_tr_dense_2k()
  hparams.encoder_layers = ["self_att", "moe_1d"] * 4
  hparams.decoder_layers = ["self_att", "enc_att", "moe_1d"] * 4
  hparams.layout = "batch:batch;experts:batch"
  hparams.moe_hidden_size = 2048
  hparams.moe_num_experts = 16
  return hparams