def imagetransformer_ae_cifar():
  """Hyperparameters for CIFAR-10 experiments."""
  hparams = transformer_ae_small()
  hparams.filter_size = 512
  hparams.num_compress_steps = 3
  hparams.startup_steps = 10000
  hparams.is_2d = 0
  hparams.learning_rate_warmup_steps = 8000
  hparams.learning_rate = 0.2
  hparams.hidden_size = 512
  hparams.batch_size = 1
  hparams.max_length = 256
  hparams.dropout = 0.0
  hparams.clip_grad_norm = 0.  # i.e. no gradient clipping
  hparams.optimizer_adam_epsilon = 1e-9
  hparams.learning_rate_decay_scheme = "noam"
  hparams.learning_rate = 0.1
  hparams.initializer_gain = 0.2
  hparams.num_hidden_layers = 6
  hparams.initializer = "uniform_unit_scaling"
  hparams.weight_decay = 0.0
  hparams.optimizer_adam_beta1 = 0.9
  hparams.optimizer_adam_beta2 = 0.98
  hparams.label_smoothing = 0.0
  hparams.norm_type = "layer"
  hparams.layer_prepostprocess_dropout = 0.0
  hparams.num_heads = 8
  hparams.task = "image"
  hparams.ffn_layer = "conv_hidden_relu"
  # All hyperparameters ending in "dropout" are automatically set to 0.0
  # when not in training mode.
  hparams.attention_dropout = 0.0
  hparams.relu_dropout = 0.
  hparams.pos = "timing"  # timing, none
  hparams.nbr_decoder_problems = 1
  hparams.num_output_layers = 3
  # TODO(trandustin): semhash doesn't work if filter_size != hidden_size. For
  # now, set default to dvq.
  hparams.bottleneck_kind = "dvq"
  hparams.add_hparam("block_size", 1)

  # dilated attention based flags
  hparams.add_hparam("gap_sizes", [2, 4, 8, 16, 32, 64, 2, 4, 8, 16, 32, 64])
  hparams.add_hparam("dilated_attention", False)

  # image size related flags
  # assuming that the image has same height and width
  hparams.add_hparam("img_len", 32)
  hparams.add_hparam("num_channels", 3)
  # Local attention params
  hparams.add_hparam("local_and_global_att", False)
  hparams.add_hparam("block_length", 256)
  hparams.add_hparam("block_width", 128)
  hparams.num_encoder_layers = 4
  hparams.num_decoder_layers = 12
  hparams.add_hparam("dec_attention_type", cia.AttentionType.LOCAL_1D)
  hparams.add_hparam("block_raster_scan", False)
  hparams.add_hparam("shared_rel", False)

  # multipos attention params
  hparams.add_hparam("q_filter_width", 1)
  hparams.add_hparam("kv_filter_width", 1)

  hparams.add_hparam("unconditional", False)  # unconditional generation

  hparams.bottom["targets"] = modalities.image_channel_embeddings_bottom
  hparams.top["targets"] = modalities.image_channel_embeddings_top
  hparams.drop_inputs = True
  hparams.do_attend_compress = False
  hparams.do_attend_decompress = False
  return hparams
def imagetransformer_ae_cifar ( ) : hparams = transformer_ae_small ( ) hparams . filter_size = 512 hparams . num_compress_steps = 3 hparams . startup_steps = 10000 hparams . is_2d = 0 hparams . learning_rate_warmup_steps = 8000 hparams . learning_rate = 0.2 hparams . hidden_size = 512 hparams . batch_size = 1 hparams . max_length = 256 hparams . dropout = 0.0 hparams . clip_grad_norm = 0. hparams . optimizer_adam_epsilon = 1e-9 hparams . learning_rate_decay_scheme = "noam" hparams . learning_rate = 0.1 hparams . initializer_gain = 0.2 hparams . num_hidden_layers = 6 hparams . initializer = "uniform_unit_scaling" hparams . weight_decay = 0.0 hparams . optimizer_adam_beta1 = 0.9 hparams . optimizer_adam_beta2 = 0.98 hparams . label_smoothing = 0.0 hparams . norm_type = "layer" hparams . layer_prepostprocess_dropout = 0.0 hparams . num_heads = 8 hparams . task = "image" hparams . ffn_layer = "conv_hidden_relu" hparams . attention_dropout = 0.0 hparams . relu_dropout = 0. hparams . pos = "timing" hparams . nbr_decoder_problems = 1 hparams . num_output_layers = 3 hparams . bottleneck_kind = "dvq" hparams . add_hparam ( "block_size" , 1 ) hparams . add_hparam ( "gap_sizes" , [ 2 , 4 , 8 , 16 , 32 , 64 , 2 , 4 , 8 , 16 , 32 , 64 ] ) hparams . add_hparam ( "dilated_attention" , False ) hparams . add_hparam ( "img_len" , 32 ) hparams . add_hparam ( "num_channels" , 3 ) hparams . add_hparam ( "local_and_global_att" , False ) hparams . add_hparam ( "block_length" , 256 ) hparams . add_hparam ( "block_width" , 128 ) hparams . num_encoder_layers = 4 hparams . num_decoder_layers = 12 hparams . add_hparam ( "dec_attention_type" , cia . AttentionType . LOCAL_1D ) hparams . add_hparam ( "block_raster_scan" , False ) hparams . add_hparam ( "shared_rel" , False ) hparams . add_hparam ( "q_filter_width" , 1 ) hparams . add_hparam ( "kv_filter_width" , 1 ) hparams . add_hparam ( "unconditional" , False ) hparams . bottom [ "targets" ] = modalities . image_channel_embeddings_bottom hparams . top [ "targets" ] = modalities . image_channel_embeddings_top hparams . drop_inputs = True hparams . do_attend_compress = False hparams . do_attend_decompress = False return hparams
Hyperparameters for CIFAR-10 experiments.