def masked_local_attention_2d(q,
                              k,
                              v,
                              query_shape=(8, 16),
                              memory_flange=(8, 16),
                              name=None):
  """Strided block local self-attention.

  Each position in a query block can attend to all the generated queries in
  the query block, which are generated in raster scan, and positions that are
  generated to the left and top. The shapes are specified by query shape and
  memory flange. Note that if you're using this function, you do not need to
  right shift. Right shifting happens inside this function separately for each
  block.

  Args:
    q: a Tensor with shape [batch, heads, h, w, depth_k]
    k: a Tensor with shape [batch, heads, h, w, depth_k]
    v: a Tensor with shape [batch, heads, h, w, depth_v]. In the current
      implementation, depth_v must be equal to depth_k.
    query_shape: an tuple indicating the height and width of each query block.
      query_shape = block_shape
    memory_flange: an integer indicating how much to look in height and width
      from each query block.
      memory shape = query_shape + (block_flange[0], 2*block_flange[1])
    name: an optional string

  Returns:
    a Tensor of shape [batch, heads, h, w, depth_v]
  """
  with tf.variable_scope(
      name, default_name="local_masked_self_attention_2d", values=[q, k, v]):
    v_shape = common_layers.shape_list(v)

    # Pad query to ensure multiple of corresponding lengths.
    q = pad_to_multiple_2d(q, query_shape)

    # Set up query blocks.
    q_indices = gather_indices_2d(q, query_shape, query_shape)
    q_new = gather_blocks_2d(q, q_indices)

    # Set up key and value blocks.
    k_flange, k_center = get_memory_region(k, query_shape, memory_flange,
                                           q_indices)
    v_flange, v_center = get_memory_region(v, query_shape, memory_flange,
                                           q_indices)
    if k_flange is not None:
      k_new = tf.concat([k_flange, k_center], axis=3)
      v_new = tf.concat([v_flange, v_center], axis=3)
    else:
      k_new = k_center
      v_new = v_center

    # Set up the masks.
    query_elements = np.prod(query_shape)
    padding_mask = None
    if k_flange is not None:
      padding_mask = tf.expand_dims(
          embedding_to_padding(k_flange) * -1e9, axis=-2)
      padding_mask = tf.tile(padding_mask, [1, 1, 1, query_elements, 1])

    center_attention_bias = attention_bias_lower_triangle(
        np.prod(query_elements))
    center_attention_bias = tf.reshape(
        center_attention_bias, [1, 1, 1, query_elements, query_elements])
    v_center_shape = common_layers.shape_list(v_center)
    center_attention_bias = tf.tile(
        center_attention_bias,
        [v_center_shape[0], v_center_shape[1], v_center_shape[2], 1, 1])
    if padding_mask is not None:
      # Combine the mask for padding and visible region.
      attention_bias = tf.concat([padding_mask, center_attention_bias], axis=4)
    else:
      attention_bias = center_attention_bias

    output = dot_product_attention(
        q_new,
        k_new,
        v_new,
        attention_bias,
        dropout_rate=0.,
        name="masked_local_2d",
        make_image_summary=False)
    # Put representations back into original shapes.
    padded_q_shape = common_layers.shape_list(q)
    output = scatter_blocks_2d(output, q_indices, padded_q_shape)

    # Remove the padding if introduced.
    output = tf.slice(output, [0, 0, 0, 0, 0],
                      [-1, -1, v_shape[2], v_shape[3], -1])
    return output
def masked_local_attention_2d ( q , k , v , query_shape = ( 8 , 16 ) , memory_flange = ( 8 , 16 ) , name = None ) : with tf . variable_scope ( name , default_name = "local_masked_self_attention_2d" , values = [ q , k , v ] ) : v_shape = common_layers . shape_list ( v ) q = pad_to_multiple_2d ( q , query_shape ) q_indices = gather_indices_2d ( q , query_shape , query_shape ) q_new = gather_blocks_2d ( q , q_indices ) k_flange , k_center = get_memory_region ( k , query_shape , memory_flange , q_indices ) v_flange , v_center = get_memory_region ( v , query_shape , memory_flange , q_indices ) if k_flange is not None : k_new = tf . concat ( [ k_flange , k_center ] , axis = 3 ) v_new = tf . concat ( [ v_flange , v_center ] , axis = 3 ) else : k_new = k_center v_new = v_center query_elements = np . prod ( query_shape ) padding_mask = None if k_flange is not None : padding_mask = tf . expand_dims ( embedding_to_padding ( k_flange ) * - 1e9 , axis = - 2 ) padding_mask = tf . tile ( padding_mask , [ 1 , 1 , 1 , query_elements , 1 ] ) center_attention_bias = attention_bias_lower_triangle ( np . prod ( query_elements ) ) center_attention_bias = tf . reshape ( center_attention_bias , [ 1 , 1 , 1 , query_elements , query_elements ] ) v_center_shape = common_layers . shape_list ( v_center ) center_attention_bias = tf . tile ( center_attention_bias , [ v_center_shape [ 0 ] , v_center_shape [ 1 ] , v_center_shape [ 2 ] , 1 , 1 ] ) if padding_mask is not None : attention_bias = tf . concat ( [ padding_mask , center_attention_bias ] , axis = 4 ) else : attention_bias = center_attention_bias output = dot_product_attention ( q_new , k_new , v_new , attention_bias , dropout_rate = 0. , name = "masked_local_2d" , make_image_summary = False ) padded_q_shape = common_layers . shape_list ( q ) output = scatter_blocks_2d ( output , q_indices , padded_q_shape ) output = tf . slice ( output , [ 0 , 0 , 0 , 0 , 0 ] , [ - 1 , - 1 , v_shape [ 2 ] , v_shape [ 3 ] , - 1 ] ) return output
Strided block local self-attention.

  Each position in a query block can attend to all the generated queries in
  the query block, which are generated in raster scan, and positions that are
  generated to the left and top. The shapes are specified by query shape and
  memory flange. Note that if you're using this function, you do not need to
  right shift. Right shifting happens inside this function separately for each
  block.

  Args:
    q: a Tensor with shape [batch, heads, h, w, depth_k]
    k: a Tensor with shape [batch, heads, h, w, depth_k]
    v: a Tensor with shape [batch, heads, h, w, depth_v]. In the current
      implementation, depth_v must be equal to depth_k.
    query_shape: an tuple indicating the height and width of each query block.
      query_shape = block_shape
    memory_flange: an integer indicating how much to look in height and width
      from each query block.
      memory shape = query_shape + (block_flange[0], 2*block_flange[1])
    name: an optional string

  Returns:
    a Tensor of shape [batch, heads, h, w, depth_v]