Generator for parsing as a sequence-to-sequence task that uses tokens.

  This generator assumes the files parsing_{train,dev}.trees, which contain
  trees in WSJ format.

  Args:
    data_dir: path to the data directory.
    tmp_dir: path to temporary storage directory.
    train: whether we're training or not.
    source_vocab_size: source vocab size.
    target_vocab_size: target vocab size.

  Returns:
    A generator to a dictionary of inputs and outputs.

def parsing_token_generator(data_dir, tmp_dir, train, source_vocab_size,
                            target_vocab_size):
  """Generator for parsing as a sequence-to-sequence task that uses tokens.

  This generator assumes the files parsing_{train,dev}.trees, which contain
  trees in WSJ format.

  Args:
    data_dir: path to the data directory.
    tmp_dir: path to temporary storage directory.
    train: whether we're training or not.
    source_vocab_size: source vocab size.
    target_vocab_size: target vocab size.

  Returns:
    A generator to a dictionary of inputs and outputs.
  """
  # TODO(lukaszkaiser): Correct these calls to generate vocabularies. No data
  # sources are being passed.
  del (data_dir, tmp_dir, train, source_vocab_size, target_vocab_size)
  assert False, "Vocabulary generation not implemented"