def _get_language_modeling_inputs(filename,
                                  delimiter="\n",
                                  repeat=1,
                                  append_space_to_final_punctionation=True):
  """Read a file of partial texts to continue.

  The purpose of append_space_to_final_punctionation is that SubwordTokenizer
  groups punctuation and the ensuing space in the same token.  Adding a space
  causes the token to be completed.

  Args:
    filename: a string
    delimiter: a string
    repeat: an integer - we repeat the entire file that many times.
    append_space_to_final_punctionation: a boolean

  Returns:
    a list of strings
  """
  with tf.gfile.Open(filename) as f:
    text = f.read()
  inputs = text.split(delimiter)
  if not inputs[-1]:
    inputs.pop()
  inputs *= repeat
  if append_space_to_final_punctionation:
    inputs = [
        s + " " if s and s[-1] in string.punctuation else s for s in inputs]
  return inputs
def _get_language_modeling_inputs ( filename , delimiter = "\n" , repeat = 1 , append_space_to_final_punctionation = True ) : with tf . gfile . Open ( filename ) as f : text = f . read ( ) inputs = text . split ( delimiter ) if not inputs [ - 1 ] : inputs . pop ( ) inputs *= repeat if append_space_to_final_punctionation : inputs = [ s + " " if s and s [ - 1 ] in string . punctuation else s for s in inputs ] return inputs
Read a file of partial texts to continue.

  The purpose of append_space_to_final_punctionation is that SubwordTokenizer
  groups punctuation and the ensuing space in the same token.  Adding a space
  causes the token to be completed.

  Args:
    filename: a string
    delimiter: a string
    repeat: an integer - we repeat the entire file that many times.
    append_space_to_final_punctionation: a boolean

  Returns:
    a list of strings