def map_file(filename, lang, vocab_size, vocab_file_prefix):
  input_file = filename + '.' + lang
  vocab_file = vocab_file_prefix + '.' + str(vocab_size) + '.vocab.' + lang
  (vocab_map, v) = io_vocab.get_vocab(input_file, vocab_file, -1, vocab_size)
  print "# vocab size is %d" % v
  output_file = filename + '.' + str(vocab_size) + '.id.' + lang
  sentences = io_vocab.get_mapped_sentence(input_file, vocab_map, output_file)
  max_len = max([len(x) for x in sentences])
  print "# max length of sentence is %d" % max_len
Helper function to control the loop in tests
dataset/ETHPy150Open yuhaozhang/nnjm-global/code/io_preprocess.py/map_file--crossover with--dataset/ETHPy150Open bretth/django-pq/pq/worker.py/Worker._dequeue_loop