Aggregate stats in per-shard stats files.

def aggregate_stats(stats_files):
  """Aggregate stats in per-shard stats files."""
  all_stats = {}
  for fname in stats_files:
    with tf.gfile.Open(fname) as f:
      stats = json.loads(f.read())
      for k, v in stats.iteritems():
        if k not in all_stats:
          if isinstance(v, list):
            all_stats[k] = []
          else:
            all_stats[k] = 0

        if isinstance(v, list):
          all_stats[k].extend(v)
        else:
          all_stats[k] += v

  stats = all_stats
  ref_coverage = float(stats["total_found_refs"]) / stats["total_original_refs"]
  len_bounds = [0, 2, 10, 100, 1000, 5000, 10000, 20000, 50000, 100000, 1000000]
  len_counts, len_bounds = np.histogram(stats["ref_lengths"], len_bounds)
  len_dist = len_counts.astype(np.float32) / len_counts.sum()
  wiki_coverage = (float(stats["num_wikis_written"]) /
                   stats["total_original_wikis"])
  wikis_skipped_no_ref = (float(stats["wikis_skipped_no_refs"]) /
                          stats["total_original_wikis"])
  wikis_skipped_no_lead = (float(stats["wikis_skipped_short_lead"]) /
                           stats["total_original_wikis"])
  wiki_ref_coverage = [
      float(found) / orig for found, orig
      in zip(stats["wiki_found_refs"], stats["wiki_original_refs"]) if found
  ]
  coverage_bounds = np.arange(21).astype(np.float32) / 20
  coverage_counts, coverage_bounds = np.histogram(wiki_ref_coverage,
                                                  coverage_bounds)
  coverage_dist = coverage_counts.astype(np.float32) / coverage_counts.sum()

  agg_stats = dict(
      total_original_wikis=stats["total_original_wikis"],
      total_original_refs=stats["total_original_refs"],
      wiki_coverage=wiki_coverage,
      wikis_skipped_no_ref=wikis_skipped_no_ref,
      wikis_skipped_no_lead=wikis_skipped_no_lead,
      overall_ref_coverage=ref_coverage,
      per_wiki_ref_coverage_dist=list((coverage_dist * 100).astype(int)),
      per_wiki_ref_coverage_bounds=list((coverage_bounds * 100).astype(int)),
      ref_len_dist=list((len_dist * 100).astype(int)),
      ref_len_bounds=list(len_bounds),
  )
  return agg_stats