def tokenize(self, char_input, max_len, by_char=False):
    """Produces the list of integer indices corresponding to a token list.

    Args:
      char_input: The character string to be tokenized.
      max_len: Truncation length.
      by_char: If true each character is a token - otherwise alpha-numeric
               groupings are tokens.

    Returns:
      A padded list of string tokens and the true sequence length.

    Raises:
      ValueError: the token sequence is too long.
    """
    if by_char:
      tokenized_list = [self._vocab_dict[token] for token in char_input]
    else:
      tokenized_list = []
      compound_token = ""
      for token in char_input:
        # Compose alphanumeric inputs into compound tokens.
        add_number = compound_token.isdigit() and not token.isdigit()
        add_word = compound_token.isalpha() and not token.isalpha()
        if add_number or add_word:
          tokenized_list.append(self._vocab_dict[compound_token])
          compound_token = ""
        # Add token or build compound token.
        if token.isdigit():
          compound_token += token
        elif token.isalpha():
          compound_token += token
        else:
          tokenized_list.append(self._vocab_dict[token])
      if compound_token:
        tokenized_list.append(self._vocab_dict[compound_token])

    # To ensure uniform batch sequence length pad the sequence.
    seq_size = len(tokenized_list)
    if seq_size < max_len:
      padding = [self._vocab_dict[get_padding()]] * (max_len - seq_size)
      tokenized_list.extend(padding)
    elif seq_size > max_len:
      raise ValueError("Token sequence is too large: {}".format(
          len(tokenized_list)))

    return tokenized_list, seq_size
def tokenize ( self , char_input , max_len , by_char = False ) : if by_char : tokenized_list = [ self . _vocab_dict [ token ] for token in char_input ] else : tokenized_list = [ ] compound_token = "" for token in char_input : add_number = compound_token . isdigit ( ) and not token . isdigit ( ) add_word = compound_token . isalpha ( ) and not token . isalpha ( ) if add_number or add_word : tokenized_list . append ( self . _vocab_dict [ compound_token ] ) compound_token = "" if token . isdigit ( ) : compound_token += token elif token . isalpha ( ) : compound_token += token else : tokenized_list . append ( self . _vocab_dict [ token ] ) if compound_token : tokenized_list . append ( self . _vocab_dict [ compound_token ] ) seq_size = len ( tokenized_list ) if seq_size < max_len : padding = [ self . _vocab_dict [ get_padding ( ) ] ] * ( max_len - seq_size ) tokenized_list . extend ( padding ) elif seq_size > max_len : raise ValueError ( "Token sequence is too large: {}" . format ( len ( tokenized_list ) ) ) return tokenized_list , seq_size
Produces the list of integer indices corresponding to a token list.

    Args:
      char_input: The character string to be tokenized.
      max_len: Truncation length.
      by_char: If true each character is a token - otherwise alpha-numeric
               groupings are tokens.

    Returns:
      A padded list of string tokens and the true sequence length.

    Raises:
      ValueError: the token sequence is too long.