Prepare tokens for encoding.

  Tokens followed by a single space have "_" appended and the single space token
  is dropped.

  If a token is _UNDERSCORE_REPLACEMENT, it is broken up into 2 tokens.

  Args:
    tokens: `list<str>`, tokens to prepare.

  Returns:
    `list<str>` prepared tokens.

def _prepare_tokens_for_encode(tokens):
  """Prepare tokens for encoding.

  Tokens followed by a single space have "_" appended and the single space token
  is dropped.

  If a token is _UNDERSCORE_REPLACEMENT, it is broken up into 2 tokens.

  Args:
    tokens: `list<str>`, tokens to prepare.

  Returns:
    `list<str>` prepared tokens.
  """
  prepared_tokens = []

  def _prepare_token(t, next_t):
    skip_next = False
    t = _escape(t)
    # If next token is a single space, add _ suffix to token and skip the
    # empty space.
    if next_t == " ":
      t += "_"
      skip_next = True
    return t, skip_next

  next_tokens = tokens[1:] + [None]
  skip_single_token = False
  for token, next_token in zip(tokens, next_tokens):
    if skip_single_token:
      skip_single_token = False
      continue

    # If the user-supplied string contains the underscore replacement string,
    # break it into 2 tokens and encode those separately.
    if token == _UNDERSCORE_REPLACEMENT:
      t1, t2 = _UNDERSCORE_REPLACEMENT[:2], _UNDERSCORE_REPLACEMENT[2:]
      t1, _ = _prepare_token(t1, None)
      t2, _ = _prepare_token(t2, next_token)
      prepared_tokens.append(t1)
      prepared_tokens.append(t2)
      continue

    token, skip_single_token = _prepare_token(token, next_token)
    prepared_tokens.append(token)
  return prepared_tokens