Produces the list of integer indices corresponding to a token list.

    Args:
      char_input: The character string to be tokenized.
      max_len: Truncation length.
      by_char: If true each character is a token - otherwise alpha-numeric
               groupings are tokens.

    Returns:
      A padded list of string tokens and the true sequence length.

    Raises:
      ValueError: the token sequence is too long.

def tokenize(self, char_input, max_len, by_char=False):
    """Produces the list of integer indices corresponding to a token list.

    Args:
      char_input: The character string to be tokenized.
      max_len: Truncation length.
      by_char: If true each character is a token - otherwise alpha-numeric
               groupings are tokens.

    Returns:
      A padded list of string tokens and the true sequence length.

    Raises:
      ValueError: the token sequence is too long.
    """
    if by_char:
      tokenized_list = [self._vocab_dict[token] for token in char_input]
    else:
      tokenized_list = []
      compound_token = ""
      for token in char_input:
        # Compose alphanumeric inputs into compound tokens.
        add_number = compound_token.isdigit() and not token.isdigit()
        add_word = compound_token.isalpha() and not token.isalpha()
        if add_number or add_word:
          tokenized_list.append(self._vocab_dict[compound_token])
          compound_token = ""
        # Add token or build compound token.
        if token.isdigit():
          compound_token += token
        elif token.isalpha():
          compound_token += token
        else:
          tokenized_list.append(self._vocab_dict[token])
      if compound_token:
        tokenized_list.append(self._vocab_dict[compound_token])

    # To ensure uniform batch sequence length pad the sequence.
    seq_size = len(tokenized_list)
    if seq_size < max_len:
      padding = [self._vocab_dict[get_padding()]] * (max_len - seq_size)
      tokenized_list.extend(padding)
    elif seq_size > max_len:
      raise ValueError("Token sequence is too large: {}".format(
          len(tokenized_list)))

    return tokenized_list, seq_size