Returning inputs sorted according to decreasing length.

  This causes inputs of similar lengths to be processed in the same batch,
  facilitating early stopping for short sequences.

  Longer sequences are sorted first so that if you're going to get OOMs,
  you'll see it in the first batch.

  Args:
    filename: path to file with inputs, 1 per line.
    delimiter: str, delimits records in the file.

  Returns:
    a sorted list of inputs

def _get_sorted_inputs(filename, delimiter="\n"):
  """Returning inputs sorted according to decreasing length.

  This causes inputs of similar lengths to be processed in the same batch,
  facilitating early stopping for short sequences.

  Longer sequences are sorted first so that if you're going to get OOMs,
  you'll see it in the first batch.

  Args:
    filename: path to file with inputs, 1 per line.
    delimiter: str, delimits records in the file.

  Returns:
    a sorted list of inputs

  """
  tf.logging.info("Getting sorted inputs")
  with tf.gfile.Open(filename) as f:
    text = f.read()
    records = text.split(delimiter)
    inputs = [record.strip() for record in records]
    # Strip the last empty line.
    if not inputs[-1]:
      inputs.pop()
  input_lens = [(i, -len(line.split())) for i, line in enumerate(inputs)]
  sorted_input_lens = sorted(input_lens, key=operator.itemgetter(1))
  # We'll need the keys to rearrange the inputs back into their original order
  sorted_keys = {}
  sorted_inputs = []
  for i, (index, _) in enumerate(sorted_input_lens):
    sorted_inputs.append(inputs[index])
    sorted_keys[index] = i
  return sorted_inputs, sorted_keys