Series of architectural experiments on Translation.

  # run on 8-core setup

  119M params, einsum=0.95e13

  Returns:
    a hparams

def xmoe_tr_dense_2k():
  """Series of architectural experiments on Translation.

  # run on 8-core setup

  119M params, einsum=0.95e13

  Returns:
    a hparams
  """
  hparams = mtf_transformer2.mtf_bitransformer_base()
  hparams.encoder_layers = ["self_att", "drd"] * 4
  hparams.decoder_layers = ["self_att", "enc_att", "drd"] * 4
  hparams.batch_size = 64
  hparams.shared_embedding_and_softmax_weights = True
  hparams.mesh_shape = "batch:8"
  return hparams