Generator for parsing as a sequence-to-sequence task that uses tokens.

  This generator assumes the files at source_path and target_path have
  the same number of lines and yields dictionaries of "inputs" and "targets"
  where inputs and targets are token ids from source and target lines
  converted to integers using the token_map.

  Args:
    tree_path: path to the file with WSJ format trees, one per line.
    source_token_vocab: GenericVocabulary object for source vocabulary.
    target_token_vocab: GenericVocabulary object for target vocabulary.
    eos: integer to append at the end of each sequence (default: None).

  Yields:
    A dictionary {"inputs": source-line, "targets": target-line} where
    the lines are integer lists converted from tokens in the file lines.

def token_generator(tree_path, source_token_vocab, target_token_vocab,
                    eos=None):
  """Generator for parsing as a sequence-to-sequence task that uses tokens.

  This generator assumes the files at source_path and target_path have
  the same number of lines and yields dictionaries of "inputs" and "targets"
  where inputs and targets are token ids from source and target lines
  converted to integers using the token_map.

  Args:
    tree_path: path to the file with WSJ format trees, one per line.
    source_token_vocab: GenericVocabulary object for source vocabulary.
    target_token_vocab: GenericVocabulary object for target vocabulary.
    eos: integer to append at the end of each sequence (default: None).

  Yields:
    A dictionary {"inputs": source-line, "targets": target-line} where
    the lines are integer lists converted from tokens in the file lines.
  """
  eos_list = [] if eos is None else [eos]
  with tf.gfile.GFile(tree_path, mode="r") as tree_file:
    tree_line = tree_file.readline()
    while tree_line:
      source, target = words_and_tags_from_wsj_tree(tree_line)
      source_ints = source_token_vocab.encode(source.strip()) + eos_list
      target_ints = target_token_vocab.encode(target.strip()) + eos_list
      yield {"inputs": source_ints, "targets": target_ints}
      tree_line = tree_file.readline()