def get_or_generate_vocabulary(data_dir,
                               tmp_dir,
                               data_prefix,
                               max_page_size_exp,
                               approx_vocab_size=32768,
                               strip=True):
  """Get or generate the vocabulary.

  Args:
    data_dir: a string
    tmp_dir: a string
    data_prefix: a string
    max_page_size_exp: an integer
    approx_vocab_size: an integer
    strip: a boolean

  Returns:
    a TextEncoder
  """
  num_pages_for_vocab_generation = approx_vocab_size // 3
  vocab_file = vocab_filename(approx_vocab_size, strip)

  def my_generator(data_prefix):
    """Line generator for vocab."""
    count = 0
    for page in corpus_page_generator(
        all_corpus_files(data_prefix)[::-1], tmp_dir, max_page_size_exp):
      revisions = page["revisions"]
      if revisions:
        text = get_text(revisions[-1], strip=strip)
        yield text
        count += 1
        if count % 100 == 0:
          tf.logging.info("reading pages for vocab %d" % count)
        if count > num_pages_for_vocab_generation:
          break

  return generator_utils.get_or_generate_vocab_inner(data_dir, vocab_file,
                                                     approx_vocab_size,
                                                     my_generator(data_prefix))
def get_or_generate_vocabulary ( data_dir , tmp_dir , data_prefix , max_page_size_exp , approx_vocab_size = 32768 , strip = True ) : num_pages_for_vocab_generation = approx_vocab_size // 3 vocab_file = vocab_filename ( approx_vocab_size , strip ) def my_generator ( data_prefix ) : count = 0 for page in corpus_page_generator ( all_corpus_files ( data_prefix ) [ : : - 1 ] , tmp_dir , max_page_size_exp ) : revisions = page [ "revisions" ] if revisions : text = get_text ( revisions [ - 1 ] , strip = strip ) yield text count += 1 if count % 100 == 0 : tf . logging . info ( "reading pages for vocab %d" % count ) if count > num_pages_for_vocab_generation : break return generator_utils . get_or_generate_vocab_inner ( data_dir , vocab_file , approx_vocab_size , my_generator ( data_prefix ) )
Get or generate the vocabulary.

  Args:
    data_dir: a string
    tmp_dir: a string
    data_prefix: a string
    max_page_size_exp: an integer
    approx_vocab_size: an integer
    strip: a boolean

  Returns:
    a TextEncoder