def imagetransformer_base_10l_16h_big_dr01_moe_imgnet():
  """big 1d model for conditional image generation."""
  hparams = imagetransformer_base_10l_16h_big_dr01_imgnet()
  hparams.initializer = "orthogonal"
  hparams.learning_rate_warmup_steps = 16000
  hparams.add_hparam("moe_layers_decoder", "2,7")  # Which layer is MoE.
  hparams.moe_hidden_sizes = "4096"  # Hidden layer sizes (comma-separated).
  hparams.moe_num_experts = 64  # Number of experts in each MoE layer.
  hparams.moe_k = 4  # How many experts to use per batch element (try 2 or 4).
  hparams.moe_loss_coef = 3e-2  # MoE loss coefficient (1e-2 is usually ok).
  hparams.scheduled_sampling_prob = 0.1
  hparams.scheduled_sampling_warmup_steps = 200000
  return hparams
def imagetransformer_base_10l_16h_big_dr01_moe_imgnet ( ) : hparams = imagetransformer_base_10l_16h_big_dr01_imgnet ( ) hparams . initializer = "orthogonal" hparams . learning_rate_warmup_steps = 16000 hparams . add_hparam ( "moe_layers_decoder" , "2,7" ) hparams . moe_hidden_sizes = "4096" hparams . moe_num_experts = 64 hparams . moe_k = 4 hparams . moe_loss_coef = 3e-2 hparams . scheduled_sampling_prob = 0.1 hparams . scheduled_sampling_warmup_steps = 200000 return hparams
big 1d model for conditional image generation.