def load_vocab_from_corpus(in_file, freq, max_vocab_size):
    f = codecs.open(in_file, 'r', 'utf-8')
    # sys.stderr.write('# Loading vocab from %s ... ' % in_file)

    words = []
    vocab_map = {}
    freq_map = {}
    vocab_size = 0
    num_train_words = 0
    num_lines = 0
    for line in f:
        tokens = re.split('\s+', line.strip())
        num_train_words += len(tokens)
        for token in tokens:
            if token not in vocab_map:
                words.append(token)
                vocab_map[token] = vocab_size
                freq_map[token] = 0
                vocab_size += 1
            freq_map[token] += 1

        num_lines += 1
        if num_lines % 100000 == 0:
            sys.stderr.write(' (%d) ' % num_lines)
            # break
    f.close()
    # sys.stderr.write('\n  vocab_size=%d, num_train_words=%d, num_lines=%d\n' % (
    #     vocab_size, num_train_words, num_lines))

    # if restrictions are provided, filter out rare words
    if freq > 0 or max_vocab_size > 0:
        (words, vocab_map, freq_map, vocab_size) = update_vocab(
            words, vocab_map, freq_map, freq, max_vocab_size)
    return (words, vocab_map, freq_map, vocab_size, num_train_words, num_lines)
retrieve_result() calls retrieve() on the global ResultStore.
dataset/ETHPy150Open yuhaozhang/nnjm-global/code/io_vocab.py/load_vocab_from_corpus--crossover with--dataset/ETHPy150Open itamarst/crochet/crochet/tests/test_api.py/PublicAPITests.test_retrieve_result