Convert the dataset in to a simpler format.

  This function creates two files. One for being processed to produce a vocab
  and another to generate the data.

  Args:
    file_path: string, path to the file to parse.
    tmp_dir: string, path to the directory to output the files.
    train: bool, indicating if we are parsing the training set.

def _parse_dataset(file_path, tmp_dir, train):
  """Convert the dataset in to a simpler format.

  This function creates two files. One for being processed to produce a vocab
  and another to generate the data.

  Args:
    file_path: string, path to the file to parse.
    tmp_dir: string, path to the directory to output the files.
    train: bool, indicating if we are parsing the training set.
  """
  input_path = file_path
  file_name = 'train' if train else 'dev'
  gen_output_path = os.path.join(tmp_dir, file_name + '.txt')
  example_output_path = os.path.join(tmp_dir, _EXAMPLES_FILE)

  print('input path: ' + input_path)
  print('gen_output_path: ' + gen_output_path)
  print('example_output_path: ' + example_output_path)

  input_file = tf.gfile.Open(input_path, mode='r')
  examples = []
  for counter, line in enumerate(input_file):
    if counter == 0:  # Ignore first line since its a header.
      continue
    # Get the token and embedding vector.
    line_split = line.split('\t')

    parse1 = line_split[_PARSE1_INDEX]
    parse2 = line_split[_PARSE2_INDEX]
    consensus_label = line_split[_LABEL_INDEX]

    tokens1 = _get_tokens_and_tags(parse1)
    tokens2 = _get_tokens_and_tags(parse2)

    tokens1_str = ' '.join(tokens1)
    tokens2_str = ' '.join(tokens2)

    if consensus_label != '-':
      examples.append([tokens1_str, tokens2_str, consensus_label])

  input_file.close()

  # Output tab delimited file of lines of examples (sentence1, sentence2, label)
  with tf.gfile.GFile(gen_output_path, 'w') as f:
    for tokens1_str, tokens2_str, consensus_label in examples:
      f.write('%s\t%s\t%s\n' % (tokens1_str, tokens2_str, consensus_label))

  if train:
    # Output file containing all the sentences for generating the vocab from.
    with tf.gfile.GFile(example_output_path, 'w') as f:
      for tokens1_str, tokens2_str, consensus_label in examples:
        f.write('%s %s\n' % (tokens1_str, tokens2_str))