def local_attention_1d(q,
                       k,
                       v,
                       length_dim,
                       key_dim,
                       value_dim,
                       autoregressive=True,
                       length_dim_num_splits=1,
                       radius=128,
                       sequence_id=1,
                       attention_kwargs=None):
  """Attention to the a neighborood around the source.

  If autoregressive, then query position p can only see memory positions
  in the range (p - radius, p].

  If not autoregressive, then query position p can only see memory positions
  in the range (p - window_size, p + radius].

  Args:
    q: a Tensor containing length_dim
    k: a Tensor containing length_dim
    v: an optional Tensor containing length_dim.  If none then uses v=k.
    length_dim: a Dimension
    key_dim: a Dimension (the channels dimension of q and k)
    value_dim: a Dimension (the channels dimension of v)
    autoregressive: a boolean
    length_dim_num_splits: an optional integer indicating how many ways the
      length dimension is split
    radius: an integer
    sequence_id: a Tensor or an integer
    attention_kwargs: optional keyword arguments for attention()

  Returns:
    a Tensor with the shape x.shape - key_dim + value_dim

  Raises:
    ValueError: if channels or depth don't match.
  """
  # Choose a suitable block size.
  # We choose the greatest divisor of length_per_split less than or equal
  # to max(window_size, 128)
  length_per_split = length_dim.size // length_dim_num_splits
  block_length = max(radius, 128)
  while length_per_split % block_length != 0:
    block_length -= 1
  query_block_length = mtf.Dimension("query_block_length", block_length)
  memory_block_length = mtf.Dimension("memory_block_length", block_length)
  # The num_blocks dimension gets the same name as the length dimension,
  # so it will be split in the same way.
  num_blocks = mtf.Dimension(length_dim.name, length_dim.size // block_length)
  def _reshape_query(x):
    return mtf.replace_dimensions(
        x, length_dim, [num_blocks, query_block_length])
  def _reshape_memory(x):
    x = mtf.replace_dimensions(
        x, length_dim, [num_blocks, memory_block_length])
    return (mtf.left_halo_exchange if autoregressive else mtf.halo_exchange)(
        x, num_blocks, memory_block_length, radius)
  q = _reshape_query(q)
  k = _reshape_memory(k)
  if v:
    v = _reshape_memory(v)
  else:
    v = k
  if sequence_id is None:
    sequence_id = 1
  if (not isinstance(sequence_id, mtf.Tensor) or
      length_dim not in sequence_id.shape.dims):
    sequence_id += mtf.zeros(q.mesh, [length_dim], tf.int32)
  q_sequence_id = _reshape_query(sequence_id)
  m_sequence_id = _reshape_memory(sequence_id)
  pos = mtf.range(q.mesh, length_dim, dtype=tf.int32)
  q_pos = _reshape_query(pos)
  m_pos = _reshape_memory(pos)

  padded_memory_block_length = mtf.Dimension(
      "memory_block_length",
      (1 if autoregressive else 2) * radius + block_length)

  relative_position = m_pos - q_pos
  illegal = mtf.not_equal(q_sequence_id, m_sequence_id)
  illegal = mtf.logical_or(illegal, mtf.less_equal(relative_position, -radius))
  illegal = mtf.logical_or(illegal, mtf.greater(
      relative_position, 0 if autoregressive else radius))
  mask = mtf.cast(illegal, q.dtype) * -1e9
  o = attention(q, k, v, padded_memory_block_length,
                key_dim, value_dim, mask, **attention_kwargs)
  return mtf.replace_dimensions(o, [num_blocks, query_block_length], length_dim)
def local_attention_1d ( q , k , v , length_dim , key_dim , value_dim , autoregressive = True , length_dim_num_splits = 1 , radius = 128 , sequence_id = 1 , attention_kwargs = None ) : length_per_split = length_dim . size // length_dim_num_splits block_length = max ( radius , 128 ) while length_per_split % block_length != 0 : block_length -= 1 query_block_length = mtf . Dimension ( "query_block_length" , block_length ) memory_block_length = mtf . Dimension ( "memory_block_length" , block_length ) num_blocks = mtf . Dimension ( length_dim . name , length_dim . size // block_length ) def _reshape_query ( x ) : return mtf . replace_dimensions ( x , length_dim , [ num_blocks , query_block_length ] ) def _reshape_memory ( x ) : x = mtf . replace_dimensions ( x , length_dim , [ num_blocks , memory_block_length ] ) return ( mtf . left_halo_exchange if autoregressive else mtf . halo_exchange ) ( x , num_blocks , memory_block_length , radius ) q = _reshape_query ( q ) k = _reshape_memory ( k ) if v : v = _reshape_memory ( v ) else : v = k if sequence_id is None : sequence_id = 1 if ( not isinstance ( sequence_id , mtf . Tensor ) or length_dim not in sequence_id . shape . dims ) : sequence_id += mtf . zeros ( q . mesh , [ length_dim ] , tf . int32 ) q_sequence_id = _reshape_query ( sequence_id ) m_sequence_id = _reshape_memory ( sequence_id ) pos = mtf . range ( q . mesh , length_dim , dtype = tf . int32 ) q_pos = _reshape_query ( pos ) m_pos = _reshape_memory ( pos ) padded_memory_block_length = mtf . Dimension ( "memory_block_length" , ( 1 if autoregressive else 2 ) * radius + block_length ) relative_position = m_pos - q_pos illegal = mtf . not_equal ( q_sequence_id , m_sequence_id ) illegal = mtf . logical_or ( illegal , mtf . less_equal ( relative_position , - radius ) ) illegal = mtf . logical_or ( illegal , mtf . greater ( relative_position , 0 if autoregressive else radius ) ) mask = mtf . cast ( illegal , q . dtype ) * - 1e9 o = attention ( q , k , v , padded_memory_block_length , key_dim , value_dim , mask , ** attention_kwargs ) return mtf . replace_dimensions ( o , [ num_blocks , query_block_length ] , length_dim )
Attention to the a neighborood around the source.

  If autoregressive, then query position p can only see memory positions
  in the range (p - radius, p].

  If not autoregressive, then query position p can only see memory positions
  in the range (p - window_size, p + radius].

  Args:
    q: a Tensor containing length_dim
    k: a Tensor containing length_dim
    v: an optional Tensor containing length_dim.  If none then uses v=k.
    length_dim: a Dimension
    key_dim: a Dimension (the channels dimension of q and k)
    value_dim: a Dimension (the channels dimension of v)
    autoregressive: a boolean
    length_dim_num_splits: an optional integer indicating how many ways the
      length dimension is split
    radius: an integer
    sequence_id: a Tensor or an integer
    attention_kwargs: optional keyword arguments for attention()

  Returns:
    a Tensor with the shape x.shape - key_dim + value_dim

  Raises:
    ValueError: if channels or depth don't match.