Create dataset where each item is a dict of "inputs" and "targets".

  Args:
    file_pattern: String used to match the input TFRecord files.
    batch_size: Maximum number of tokens per batch of examples
    max_length: Maximum number of tokens per example
    num_cpu_cores: Number of cpu cores for parallel input processing.
    shuffle: If true, randomizes order of elements.
    repeat: Number of times to repeat the dataset. If None, the dataset is
      repeated forever.

  Returns:
    tf.data.Dataset object containing examples loaded from the files.

def _read_and_batch_from_files(
    file_pattern, batch_size, max_length, num_cpu_cores, shuffle, repeat):
  """Create dataset where each item is a dict of "inputs" and "targets".

  Args:
    file_pattern: String used to match the input TFRecord files.
    batch_size: Maximum number of tokens per batch of examples
    max_length: Maximum number of tokens per example
    num_cpu_cores: Number of cpu cores for parallel input processing.
    shuffle: If true, randomizes order of elements.
    repeat: Number of times to repeat the dataset. If None, the dataset is
      repeated forever.

  Returns:
    tf.data.Dataset object containing examples loaded from the files.
  """
  dataset = tf.data.Dataset.list_files(file_pattern)

  if shuffle:
    # Shuffle filenames
    mlperf_log.transformer_print(key=mlperf_log.INPUT_ORDER)
    dataset = dataset.shuffle(buffer_size=_FILE_SHUFFLE_BUFFER)

  # Read files and interleave results. When training, the order of the examples
  # will be non-deterministic.
  dataset = dataset.apply(
      tf.contrib.data.parallel_interleave(
          _load_records, sloppy=shuffle, cycle_length=num_cpu_cores))

  # Parse each tf.Example into a dictionary
  # TODO: Look into prefetch_input_elements for performance optimization.
  dataset = dataset.map(_parse_example,
                        num_parallel_calls=num_cpu_cores)

  # Remove examples where the input or target length exceeds the maximum length,
  dataset = dataset.filter(lambda x, y: _filter_max_length((x, y), max_length))

  # Batch such that each batch has examples of similar length.
  mlperf_log.transformer_print(key=mlperf_log.INPUT_BATCH_SIZE,
                               value=batch_size)
  mlperf_log.transformer_print(key=mlperf_log.INPUT_MAX_LENGTH,
                               value=max_length)
  dataset = _batch_examples(dataset, batch_size, max_length)
  dataset = dataset.repeat(repeat)

  # Prefetch the next element to improve speed of input pipeline.
  dataset = dataset.prefetch(1)
  return dataset