Pad left across time and pad valid across the spatial components.

  Also concats a binary feature that indicates if a feature is padded or not.

  Args:
    x: 5-D Tensor, (NTHWC)
    filter_size: list of ints
    dilations: list of ints, dilations - 1 specifies the number of holes
               between two filter elements.
  Returns:
    x_pad: 5-D Tensor.

def time_pad(x, filter_size, dilations):
  """Pad left across time and pad valid across the spatial components.

  Also concats a binary feature that indicates if a feature is padded or not.

  Args:
    x: 5-D Tensor, (NTHWC)
    filter_size: list of ints
    dilations: list of ints, dilations - 1 specifies the number of holes
               between two filter elements.
  Returns:
    x_pad: 5-D Tensor.
  """
  x_shape = common_layers.shape_list(x)
  if filter_size == [1, 1, 1]:
    return x
  _, h, w = filter_size
  eff_h = h + (h - 1)*(dilations[2] - 1)
  eff_w = w + (w - 1)*(dilations[3] - 1)
  a = (eff_h - 1) // 2  # vertical padding size
  b = (eff_w - 1) // 2  # horizontal padding size
  c = filter_size[0] - 1

  # pad across edges.
  padding = [[0, 0], [c, 0], [a, a], [b, b], [0, 0]]

  # concat a binary feature across channels to indicate a padding.
  # 1 indicates that the feature is a padding.
  x_bias = tf.zeros(x_shape[:-1] + [1])
  x_bias = tf.pad(x_bias, padding, constant_values=1)
  x_pad = tf.pad(x, padding)
  x_pad = tf.concat((x_bias, x_pad), axis=-1)
  return x_pad