Run LSTM cell with attention on inputs of shape [batch x time x size].

  Args:
    inputs: The decoder input `Tensor`, shaped `[batch_size, decoder_steps,
        hidden_size]`.
    hparams: HParams; hyperparameters.
    train: bool; `True` when constructing training graph to enable dropout.
    name: string; Create variable names under this scope.
    initial_state: Tuple of `LSTMStateTuple`s; the initial state of each layer.
    encoder_outputs: Encoder outputs; a `Tensor` shaped `[batch_size,
        encoder_steps, hidden_size]`.
    encoder_output_length: Lengths of the actual encoder outputs, excluding
        padding; a `Tensor` shaped `[batch_size]`.
    decoder_input_length: Lengths of the actual decoder inputs, excluding
        padding; a `Tensor` shaped `[batch_size]`.

  Raises:
    ValueError: If the hparams.attention_mechanism is anything other than
        luong or bahdanau.

  Returns:
    The decoder output `Tensor`, shaped `[batch_size, decoder_steps,
    hidden_size]`.

def lstm_attention_decoder(inputs, hparams, train, name, initial_state,
                           encoder_outputs, encoder_output_length,
                           decoder_input_length):
  """Run LSTM cell with attention on inputs of shape [batch x time x size].

  Args:
    inputs: The decoder input `Tensor`, shaped `[batch_size, decoder_steps,
        hidden_size]`.
    hparams: HParams; hyperparameters.
    train: bool; `True` when constructing training graph to enable dropout.
    name: string; Create variable names under this scope.
    initial_state: Tuple of `LSTMStateTuple`s; the initial state of each layer.
    encoder_outputs: Encoder outputs; a `Tensor` shaped `[batch_size,
        encoder_steps, hidden_size]`.
    encoder_output_length: Lengths of the actual encoder outputs, excluding
        padding; a `Tensor` shaped `[batch_size]`.
    decoder_input_length: Lengths of the actual decoder inputs, excluding
        padding; a `Tensor` shaped `[batch_size]`.

  Raises:
    ValueError: If the hparams.attention_mechanism is anything other than
        luong or bahdanau.

  Returns:
    The decoder output `Tensor`, shaped `[batch_size, decoder_steps,
    hidden_size]`.
  """
  layers = [_dropout_lstm_cell(hparams, train)
            for _ in range(hparams.num_hidden_layers)]
  if hparams.attention_mechanism == "luong":
    attention_mechanism_class = tf.contrib.seq2seq.LuongAttention
  elif hparams.attention_mechanism == "bahdanau":
    attention_mechanism_class = tf.contrib.seq2seq.BahdanauAttention
  else:
    raise ValueError("Unknown hparams.attention_mechanism = %s, must be "
                     "luong or bahdanau." % hparams.attention_mechanism)
  if hparams.get("max_area_width", 1) > 1:
    def _area_key_value_fn(keys, values):
      """Custom fn for computing area keys and values."""
      tf.logging.info("max_area_width=%d, area_key_mode=%s, area_value_mode=%s",
                      hparams.get("max_area_width", 1),
                      hparams.get("area_key_mode", "none"),
                      hparams.get("area_value_mode", "none"))
      keys = area_attention.compute_area_key(
          keys, max_area_width=hparams.get("max_area_width", 1),
          mode=hparams.get("area_key_mode", "none"), name="decoder_encoder",
          training=(hparams.mode == tf.estimator.ModeKeys.TRAIN))
      if hparams.get("area_value_mode", "none") == "sum":
        _, _, values, _, _ = area_attention.compute_area_features(
            values, max_area_width=hparams.get("max_area_width", 1))
      elif hparams.get("area_value_mode", "none") == "mean":
        values, _, _, _, _ = area_attention.compute_area_features(
            values, max_area_width=hparams.get("max_area_width", 1))
      else:
        raise ValueError(
            "Unsupported area_value_mode: %s" % hparams.get(
                "area_value_mode", "none"))
      return keys, values
    area_mask = area_attention.lengths_to_area_mask(
        feature_length=encoder_output_length,
        length=common_layers.shape_list(encoder_outputs)[1],
        max_area_size=hparams.get("max_area_width", "1"))
    def _area_prob_fn(score):
      alignments = tf.nn.softmax(score)
      alignments = tf.where(area_mask, alignments, tf.zeros_like(alignments))
      alignments = tf.div(alignments, tf.reduce_sum(
          alignments, axis=-1, keepdims=True))
      return alignments
    attention_mechanism = attention_mechanism_class(
        hparams.hidden_size, encoder_outputs,
        memory_sequence_length=None,
        probability_fn=_area_prob_fn,
        custom_key_value_fn=_area_key_value_fn)
  else:
    attention_mechanism = attention_mechanism_class(hparams.hidden_size,
                                                    encoder_outputs)
  cell = tf.contrib.seq2seq.AttentionWrapper(
      tf.nn.rnn_cell.MultiRNNCell(layers),
      [attention_mechanism]*hparams.num_heads,
      attention_layer_size=[hparams.attention_layer_size]*hparams.num_heads,
      output_attention=(hparams.output_attention == 1))

  batch_size = common_layers.shape_list(inputs)[0]

  initial_state = cell.zero_state(batch_size, tf.float32).clone(
      cell_state=initial_state)

  with tf.variable_scope(name):
    output, _ = tf.nn.dynamic_rnn(
        cell,
        inputs,
        decoder_input_length,
        initial_state=initial_state,
        dtype=tf.float32,
        time_major=False)
    # output is [batch_size, decoder_steps, attention_size], where
    # attention_size is either hparams.hidden_size (when
    # hparams.output_attention is 0) or hparams.attention_layer_size (when
    # hparams.output_attention is 1) times the number of attention heads.
    #
    # For multi-head attention project output back to hidden size.
    if hparams.output_attention == 1 and hparams.num_heads > 1:
      output = tf.layers.dense(output, hparams.hidden_size)

    return output