title: How to measure perplexity in tensorflow?
body: <p>I am using this tutorial code:</p>
  <pre><code>import tensorflow as tf
  
  import numpy as np
  import os
  import time
  path_to_file = tf.keras.utils.get_file('shakespeare.txt', 'https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt')
  # Read, then decode for py2 compat.
  text = open(path_to_file, 'rb').read().decode(encoding='utf-8')
  # length of text is the number of characters in it
  print('Length of text: {} characters'.format(len(text)))
  # Take a look at the first 250 characters in text
  print(text[:250])
  # The unique characters in the file
  vocab = sorted(set(text))
  print('{} unique characters'.format(len(vocab)))
  # Creating a mapping from unique characters to indices
  char2idx = {u:i for i, u in enumerate(vocab)}
  idx2char = np.array(vocab)
  
  text_as_int = np.array([char2idx[c] for c in text])
  print('{')
  for char,_ in zip(char2idx, range(20)):
      print('  {:4s}: {:3d},'.format(repr(char), char2idx[char]))
  print('  ...\n}')
  # Show how the first 13 characters from the text are mapped to integers
  print('{} ---- characters mapped to int ---- &gt; {}'.format(repr(text[:13]), text_as_int[:13]))
  # The maximum length sentence you want for a single input in characters
  # seq_length = 100
  seq_length = 50
  examples_per_epoch = len(text)//(seq_length+1)
  
  # Create training examples / targets
  char_dataset = tf.data.Dataset.from_tensor_slices(text_as_int)
  
  for i in char_dataset.take(5):
      print(idx2char[i.numpy()])
  sequences = char_dataset.batch(seq_length+1, drop_remainder=True)
  
  for item in sequences.take(5):
      print(repr(''.join(idx2char[item.numpy()])))
      
  def split_input_target(chunk):
      input_text = chunk[:-1]
      target_text = chunk[1:]
      return input_text, target_text
  
  dataset = sequences.map(split_input_target)
  
  for input_example, target_example in  dataset.take(1):
      print('Input data: ', repr(''.join(idx2char[input_example.numpy()])))
      print('Target data:', repr(''.join(idx2char[target_example.numpy()])))
  
  for i, (input_idx, target_idx) in enumerate(zip(input_example[:5], target_example[:5])):
      print(&quot;Step {:4d}&quot;.format(i))
      print(&quot;  input: {} ({:s})&quot;.format(input_idx, repr(idx2char[input_idx])))
      print(&quot;  expected output: {} ({:s})&quot;.format(target_idx, repr(idx2char[target_idx])))
  
  # Batch size
  BATCH_SIZE = 64
  
  # Buffer size to shuffle the dataset
  # (TF data is designed to work with possibly infinite sequences,
  # so it doesn't attempt to shuffle the entire sequence in memory. Instead,
  # it maintains a buffer in which it shuffles elements).
  BUFFER_SIZE = 10000
  
  dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)
  
  dataset
  
  # Length of the vocabulary in chars
  vocab_size = len(vocab)
  
  # The embedding dimension
  embedding_dim = 128
  #embedding_dim = 256
  
  # Number of RNN units
  rnn_units = 128
  #rnn_units = 1024
  
  def build_model(vocab_size, embedding_dim, rnn_units, batch_size):
      model = tf.keras.Sequential([
          tf.keras.layers.Embedding(vocab_size, embedding_dim,
                                    batch_input_shape=[batch_size, None]),
          tf.keras.layers.GRU(rnn_units,
                              return_sequences=True,
                              stateful=True,
                              recurrent_initializer='glorot_uniform'),
          tf.keras.layers.GRU(rnn_units,
                              return_sequences=True,
                              stateful=True,
                              recurrent_initializer='glorot_uniform'),
          tf.keras.layers.Dense(vocab_size)
      ])
      return model
  
  model = build_model(
      vocab_size=len(vocab),
      embedding_dim=embedding_dim,
      rnn_units=rnn_units,
      batch_size=BATCH_SIZE)
  
  
  for input_example_batch, target_example_batch in dataset.take(1):
      example_batch_predictions = model(input_example_batch)
      print(example_batch_predictions.shape, &quot;# (batch_size, sequence_length, vocab_size)&quot;)
  
  model.summary()
  
  
  
  sampled_indices = tf.random.categorical(example_batch_predictions[0], num_samples=1)
  sampled_indices = tf.squeeze(sampled_indices,axis=-1).numpy()
  
  sampled_indices
  
  print(&quot;Input: \n&quot;, repr(&quot;&quot;.join(idx2char[input_example_batch[0].numpy()])))
  print()
  print(&quot;Next Char Predictions: \n&quot;, repr(&quot;&quot;.join(idx2char[sampled_indices ])))
  
  def loss(labels, logits):
      return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)
  
  example_batch_loss = loss(target_example_batch, example_batch_predictions)
  print(&quot;Prediction shape: &quot;, example_batch_predictions.shape, &quot; # (batch_size, sequence_length, vocab_size)&quot;)
  print(&quot;scalar_loss:      &quot;, example_batch_loss.numpy().mean())
  
  model.compile(optimizer='adam', loss=loss)
  
  # Directory where the checkpoints will be saved
  checkpoint_dir = './training_checkpoints'
  # Name of the checkpoint files
  checkpoint_prefix = os.path.join(checkpoint_dir, &quot;ckpt_{epoch}&quot;)
  
  checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
      filepath=checkpoint_prefix,
      save_weights_only=True)
  
  #EPOCHS = 30
  EPOCHS = 10
  
  history = model.fit(dataset, epochs=EPOCHS, callbacks=[checkpoint_callback])
  tf.train.latest_checkpoint(checkpoint_dir)
  model = build_model(vocab_size, embedding_dim, rnn_units, batch_size=1)
  
  model.load_weights(tf.train.latest_checkpoint(checkpoint_dir))
  
  model.build(tf.TensorShape([1, None]))
  
  
  model.summary()
  </code></pre>
  <p>I have set the parameters so it runs quickly for testing. I would like to measure the <a href="https://en.wikipedia.org/wiki/Perplexity" rel="nofollow noreferrer">perplexity</a> of the model, say on the training set itself or some other test text.  How can I do that?</p>
  <hr />
  <p>To make the question completely self-contained, given the model made above, how would you compute the perplexity of the string &quot;where&quot;?</p>