def multihead_attention_2d(query_antecedent,
                           memory_antecedent,
                           total_key_depth,
                           total_value_depth,
                           output_depth,
                           num_heads,
                           attention_type="local_attention_2d",
                           query_shape=(8, 16),
                           memory_flange=(8, 16),
                           name=None):
  """2d Multihead scaled-dot-product attention with inp/output transformations.

  Args:
    query_antecedent: a Tensor with shape [batch, h, w, depth_k]
    memory_antecedent: a Tensor with shape [batch, h, w, depth_k]
    total_key_depth: an integer
    total_value_depth: an integer
    output_depth: an integer
    num_heads: an integer dividing total_key_depth and total_value_depth
    attention_type: String, type of attention function to use.
    query_shape: an tuple indicating the height and width of each query block.
    memory_flange: an integer indicating how much to look in height and width
    name: an optional string

  Returns:
    A Tensor of shape [batch, h, w, output_depth]

  Raises:
    ValueError: if the key depth or value depth are not divisible by the
      number of attention heads.
  """
  if total_key_depth % num_heads != 0:
    raise ValueError("Key depth (%d) must be divisible by the number of "
                     "attention heads (%d)." % (total_key_depth, num_heads))
  if total_value_depth % num_heads != 0:
    raise ValueError("Value depth (%d) must be divisible by the number of "
                     "attention heads (%d)." % (total_value_depth, num_heads))
  with tf.variable_scope(
      name,
      default_name="multihead_attention_2d",
      values=[query_antecedent, memory_antecedent]):
    q, k, v = compute_qkv(query_antecedent, memory_antecedent, total_key_depth,
                          total_value_depth)
    # after splitting, shape is [batch, heads, h, w, depth]
    q = split_heads_2d(q, num_heads)
    k = split_heads_2d(k, num_heads)
    v = split_heads_2d(v, num_heads)
    key_depth_per_head = total_key_depth // num_heads
    q *= key_depth_per_head**-0.5
    if attention_type == "local_attention_2d":
      x = local_attention_2d(
          q, k, v, query_shape=query_shape, memory_flange=memory_flange)
    elif attention_type == "masked_local_attention_2d":
      assert attention_type == "masked_local_attention_2d"
      x = masked_local_attention_2d(
          q, k, v, query_shape=query_shape, memory_flange=memory_flange)
    else:
      assert attention_type == "unmasked_local_attention_2d_tpu"
      x = dot_product_unmasked_attention_local_2d_tpu(
          q, k, v, None, max_relative_position=None, query_shape=query_shape)
    x = combine_heads_2d(x)
    x = common_layers.dense(
        x, output_depth, use_bias=False, name="output_transform")
    return x
def multihead_attention_2d ( query_antecedent , memory_antecedent , total_key_depth , total_value_depth , output_depth , num_heads , attention_type = "local_attention_2d" , query_shape = ( 8 , 16 ) , memory_flange = ( 8 , 16 ) , name = None ) : if total_key_depth % num_heads != 0 : raise ValueError ( "Key depth (%d) must be divisible by the number of " "attention heads (%d)." % ( total_key_depth , num_heads ) ) if total_value_depth % num_heads != 0 : raise ValueError ( "Value depth (%d) must be divisible by the number of " "attention heads (%d)." % ( total_value_depth , num_heads ) ) with tf . variable_scope ( name , default_name = "multihead_attention_2d" , values = [ query_antecedent , memory_antecedent ] ) : q , k , v = compute_qkv ( query_antecedent , memory_antecedent , total_key_depth , total_value_depth ) q = split_heads_2d ( q , num_heads ) k = split_heads_2d ( k , num_heads ) v = split_heads_2d ( v , num_heads ) key_depth_per_head = total_key_depth // num_heads q *= key_depth_per_head ** - 0.5 if attention_type == "local_attention_2d" : x = local_attention_2d ( q , k , v , query_shape = query_shape , memory_flange = memory_flange ) elif attention_type == "masked_local_attention_2d" : assert attention_type == "masked_local_attention_2d" x = masked_local_attention_2d ( q , k , v , query_shape = query_shape , memory_flange = memory_flange ) else : assert attention_type == "unmasked_local_attention_2d_tpu" x = dot_product_unmasked_attention_local_2d_tpu ( q , k , v , None , max_relative_position = None , query_shape = query_shape ) x = combine_heads_2d ( x ) x = common_layers . dense ( x , output_depth , use_bias = False , name = "output_transform" ) return x
2d Multihead scaled-dot-product attention with inp/output transformations.

  Args:
    query_antecedent: a Tensor with shape [batch, h, w, depth_k]
    memory_antecedent: a Tensor with shape [batch, h, w, depth_k]
    total_key_depth: an integer
    total_value_depth: an integer
    output_depth: an integer
    num_heads: an integer dividing total_key_depth and total_value_depth
    attention_type: String, type of attention function to use.
    query_shape: an tuple indicating the height and width of each query block.
    memory_flange: an integer indicating how much to look in height and width
    name: an optional string

  Returns:
    A Tensor of shape [batch, h, w, output_depth]

  Raises:
    ValueError: if the key depth or value depth are not divisible by the
      number of attention heads.