def get_position_encoding(
    length, hidden_size, min_timescale=1.0, max_timescale=1.0e4):
  """Return positional encoding.

  Calculates the position encoding as a mix of sine and cosine functions with
  geometrically increasing wavelengths.
  Defined and formulized in Attention is All You Need, section 3.5.

  Args:
    length: Sequence length.
    hidden_size: Size of the
    min_timescale: Minimum scale that will be applied at each position
    max_timescale: Maximum scale that will be applied at each position

  Returns:
    Tensor with shape [length, hidden_size]
  """
  position = tf.to_float(tf.range(length))
  num_timescales = hidden_size // 2
  log_timescale_increment = (
      math.log(float(max_timescale) / float(min_timescale)) /
      (tf.to_float(num_timescales) - 1))
  inv_timescales = min_timescale * tf.exp(
      tf.to_float(tf.range(num_timescales)) * -log_timescale_increment)
  scaled_time = tf.expand_dims(position, 1) * tf.expand_dims(inv_timescales, 0)
  signal = tf.concat([tf.sin(scaled_time), tf.cos(scaled_time)], axis=1)
  return signal
def get_position_encoding ( length , hidden_size , min_timescale = 1.0 , max_timescale = 1.0e4 ) : position = tf . to_float ( tf . range ( length ) ) num_timescales = hidden_size // 2 log_timescale_increment = ( math . log ( float ( max_timescale ) / float ( min_timescale ) ) / ( tf . to_float ( num_timescales ) - 1 ) ) inv_timescales = min_timescale * tf . exp ( tf . to_float ( tf . range ( num_timescales ) ) * - log_timescale_increment ) scaled_time = tf . expand_dims ( position , 1 ) * tf . expand_dims ( inv_timescales , 0 ) signal = tf . concat ( [ tf . sin ( scaled_time ) , tf . cos ( scaled_time ) ] , axis = 1 ) return signal
Return positional encoding.

  Calculates the position encoding as a mix of sine and cosine functions with
  geometrically increasing wavelengths.
  Defined and formulized in Attention is All You Need, section 3.5.

  Args:
    length: Sequence length.
    hidden_size: Size of the
    min_timescale: Minimum scale that will be applied at each position
    max_timescale: Maximum scale that will be applied at each position

  Returns:
    Tensor with shape [length, hidden_size]