Generate a PR curve with precision and recall evenly weighted.

  Arguments:
    logdir: The directory into which to store all the runs' data.
    steps: The number of steps to run for.
    run_name: The name of the run.
    thresholds: The number of thresholds to use for PR curves.
    mask_every_other_prediction: Whether to mask every other prediction by
      alternating weights between 0 and 1.

def start_runs(
    logdir,
    steps,
    run_name,
    thresholds,
    mask_every_other_prediction=False):
  """Generate a PR curve with precision and recall evenly weighted.

  Arguments:
    logdir: The directory into which to store all the runs' data.
    steps: The number of steps to run for.
    run_name: The name of the run.
    thresholds: The number of thresholds to use for PR curves.
    mask_every_other_prediction: Whether to mask every other prediction by
      alternating weights between 0 and 1.
  """
  tf.compat.v1.reset_default_graph()
  tf.compat.v1.set_random_seed(42)

  # Create a normal distribution layer used to generate true color labels.
  distribution = tf.compat.v1.distributions.Normal(loc=0., scale=142.)

  # Sample the distribution to generate colors. Lets generate different numbers
  # of each color. The first dimension is the count of examples.

  # The calls to sample() are given fixed random seed values that are "magic"
  # in that they correspond to the default seeds for those ops when the PR
  # curve test (which depends on this code) was written. We've pinned these
  # instead of continuing to use the defaults since the defaults are based on
  # node IDs from the sequence of nodes added to the graph, which can silently
  # change when this code or any TF op implementations it uses are modified.

  # TODO(nickfelt): redo the PR curve test to avoid reliance on random seeds.

  # Generate reds.
  number_of_reds = 100
  true_reds = tf.clip_by_value(
      tf.concat([
          255 - tf.abs(distribution.sample([number_of_reds, 1], seed=11)),
          tf.abs(distribution.sample([number_of_reds, 2], seed=34))
      ], axis=1),
      0, 255)

  # Generate greens.
  number_of_greens = 200
  true_greens = tf.clip_by_value(
      tf.concat([
          tf.abs(distribution.sample([number_of_greens, 1], seed=61)),
          255 - tf.abs(distribution.sample([number_of_greens, 1], seed=82)),
          tf.abs(distribution.sample([number_of_greens, 1], seed=105))
      ], axis=1),
      0, 255)

  # Generate blues.
  number_of_blues = 150
  true_blues = tf.clip_by_value(
      tf.concat([
          tf.abs(distribution.sample([number_of_blues, 2], seed=132)),
          255 - tf.abs(distribution.sample([number_of_blues, 1], seed=153))
      ], axis=1),
      0, 255)

  # Assign each color a vector of 3 booleans based on its true label.
  labels = tf.concat([
      tf.tile(tf.constant([[True, False, False]]), (number_of_reds, 1)),
      tf.tile(tf.constant([[False, True, False]]), (number_of_greens, 1)),
      tf.tile(tf.constant([[False, False, True]]), (number_of_blues, 1)),
  ], axis=0)

  # We introduce 3 normal distributions. They are used to predict whether a
  # color falls under a certain class (based on distances from corners of the
  # color triangle). The distributions vary per color. We have the distributions
  # narrow over time.
  initial_standard_deviations = [v + FLAGS.steps for v in (158, 200, 242)]
  iteration = tf.compat.v1.placeholder(tf.int32, shape=[])
  red_predictor = tf.compat.v1.distributions.Normal(
      loc=0.,
      scale=tf.cast(
          initial_standard_deviations[0] - iteration,
          dtype=tf.float32))
  green_predictor = tf.compat.v1.distributions.Normal(
      loc=0.,
      scale=tf.cast(
          initial_standard_deviations[1] - iteration,
          dtype=tf.float32))
  blue_predictor = tf.compat.v1.distributions.Normal(
      loc=0.,
      scale=tf.cast(
          initial_standard_deviations[2] - iteration,
          dtype=tf.float32))

  # Make predictions (assign 3 probabilities to each color based on each color's
  # distance to each of the 3 corners). We seek double the area in the right
  # tail of the normal distribution.
  examples = tf.concat([true_reds, true_greens, true_blues], axis=0)
  probabilities_colors_are_red = (1 - red_predictor.cdf(
      tf.norm(tensor=examples - tf.constant([255., 0, 0]), axis=1))) * 2
  probabilities_colors_are_green = (1 - green_predictor.cdf(
      tf.norm(tensor=examples - tf.constant([0, 255., 0]), axis=1))) * 2
  probabilities_colors_are_blue = (1 - blue_predictor.cdf(
      tf.norm(tensor=examples - tf.constant([0, 0, 255.]), axis=1))) * 2

  predictions = (
      probabilities_colors_are_red,
      probabilities_colors_are_green,
      probabilities_colors_are_blue
  )

  # This is the crucial piece. We write data required for generating PR curves.
  # We create 1 summary per class because we create 1 PR curve per class.
  for i, color in enumerate(('red', 'green', 'blue')):
    description = ('The probabilities used to create this PR curve are '
                   'generated from a normal distribution. Its standard '
                   'deviation is initially %0.0f and decreases over time.' %
                   initial_standard_deviations[i])

    weights = None
    if mask_every_other_prediction:
      # Assign a weight of 0 to every even-indexed prediction. Odd-indexed
      # predictions are assigned a default weight of 1.
      consecutive_indices = tf.reshape(
          tf.range(tf.size(input=predictions[i])), tf.shape(input=predictions[i]))
      weights = tf.cast(consecutive_indices % 2, dtype=tf.float32)

    summary.op(
        name=color,
        labels=labels[:, i],
        predictions=predictions[i],
        num_thresholds=thresholds,
        weights=weights,
        display_name='classifying %s' % color,
        description=description)
  merged_summary_op = tf.compat.v1.summary.merge_all()
  events_directory = os.path.join(logdir, run_name)
  sess = tf.compat.v1.Session()
  writer = tf.compat.v1.summary.FileWriter(events_directory, sess.graph)

  for step in xrange(steps):
    feed_dict = {
        iteration: step,
    }
    merged_summary = sess.run(merged_summary_op, feed_dict=feed_dict)
    writer.add_summary(merged_summary, step)

  writer.close()