With compressed attention.

def xmoe2_v1_l4k_compressed_c4():
  """With compressed attention."""
  hparams = xmoe2_v1_l4k()
  hparams.decoder_layers = [
      "compressed_att" if l == "att" else l for l in hparams.decoder_layers]
  hparams.compression_factor = 4
  return hparams