original_docstring: 
      Convert a document into ids based on given vocabulary
      :param Dict[str,int] word_to_id: Word to ids vocabulary to use for ids.
      :param Iterator[str] reader:
      :param Callable tokenizer:
      :param Optional[str] unknown_token: The token to use for tokens that are not in the index. If
      None is given then '<OOV>' is used.
      :param Optional[str] pad_token: The token to use for tokens that are not in the index. If
      None is given then '<PAD>' is used.
      :return:
      
docstring: Convert a document into ids based on given vocabulary
docstring_tokens: Convert a document into ids based on given vocabulary
code: def convert_text_to_padded_ids(word_to_id, reader, tokenizer, unknown_token=None, pad_token=None,
                                 **kwargs):
      """
      Convert a document into ids based on given vocabulary
      :param Dict[str,int] word_to_id: Word to ids vocabulary to use for ids.
      :param Iterator[str] reader:
      :param Callable tokenizer:
      :param Optional[str] unknown_token: The token to use for tokens that are not in the index. If
      None is given then '<OOV>' is used.
      :param Optional[str] pad_token: The token to use for tokens that are not in the index. If
      None is given then '<PAD>' is used.
      :return:
      """
      assert len(kwargs) == 1
      seq_lengths = kwargs[list(kwargs.keys())[0]]
  
      if pad_token is None:
          pad_token = '<PAD>'
  
      max_row_length = seq_lengths['max_row_length']
      max_sent_length = seq_lengths['max_sent_length'] if 'max_sent_length' in seq_lengths else None
      for row_label, row_text in reader:
          tokens = tokenizer(row_text)
          if 'max_sent_length' in seq_lengths:
              tokens = split_and_padd_sentences(row_text, max_sent_length, max_row_length, tokenizer, pad_token)
          else:
              if len(tokens) < max_row_length:
                  tokens += [pad_token] * (max_row_length - len(tokens))
          token_ids = convert_tokens_to_ids(word_to_id, tokens, max_sent_length=max_sent_length,
                                            unknown_token=unknown_token)
  
          token_ids = token_ids[:max_row_length]
  
          if any(isinstance(el, list) for el in token_ids):
              token_ids = [item for sublist in token_ids for item in sublist]
  
          yield row_label, token_ids
code_tokens: def convert_text_to_padded_ids ( word_to_id , reader , tokenizer , unknown_token = None , pad_token = None , ** kwargs ) : assert len ( kwargs ) == 1 seq_lengths = kwargs [ list ( kwargs . keys ( ) ) [ 0 ] ] if pad_token is None : pad_token = '<PAD>' max_row_length = seq_lengths [ 'max_row_length' ] max_sent_length = seq_lengths [ 'max_sent_length' ] if 'max_sent_length' in seq_lengths else None for row_label , row_text in reader : tokens = tokenizer ( row_text ) if 'max_sent_length' in seq_lengths : tokens = split_and_padd_sentences ( row_text , max_sent_length , max_row_length , tokenizer , pad_token ) else : if len ( tokens ) < max_row_length : tokens += [ pad_token ] * ( max_row_length - len ( tokens ) ) token_ids = convert_tokens_to_ids ( word_to_id , tokens , max_sent_length = max_sent_length , unknown_token = unknown_token ) token_ids = token_ids [ : max_row_length ] if any ( isinstance ( el , list ) for el in token_ids ) : token_ids = [ item for sublist in token_ids for item in sublist ] yield row_label , token_ids
short_docstring: Convert a document into ids based on given vocabulary
short_docstring_tokens: Convert a document into ids based on given vocabulary
comment: """
      Convert a document into ids based on given vocabulary
      :param Dict[str,int] word_to_id: Word to ids vocabulary to use for ids.
      :param Iterator[str] reader:
      :param Callable tokenizer:
      :param Optional[str] unknown_token: The token to use for tokens that are not in the index. If
      None is given then '<OOV>' is used.
      :param Optional[str] pad_token: The token to use for tokens that are not in the index. If
      None is given then '<PAD>' is used.
      :return:
      """