src/main.cpp
// Read the vocabulary file; create the dictionary and huffman_encoder according opt

Read the vocabulary file; create the dictionary and huffman_encoder according opt
Read the vocabulary file ; create the dictionary and huffman_encoder according opt
int64_t LoadVocab(Option *opt, Dictionary *dictionary, HuffmanEncoder *huffman_encoder)
{
	int64_t total_words = 0;
	char word[MAX_STRING];
	FILE* fid = nullptr;
	printf("vocab_file %s\n", opt->read_vocab_file);
	if (opt->read_vocab_file != nullptr && strlen(opt->read_vocab_file) > 0)
	{
		printf("Begin to load vocabulary file [%s] ...\n", opt->read_vocab_file);
		fid = fopen(opt->read_vocab_file, "r");
		int word_freq;
		while (fscanf(fid, "%s %d", word, &word_freq) != EOF)
		{
			dictionary->Insert(word, word_freq);
		}
	}
	dictionary->RemoveWordsLessThan(opt->min_count);
	printf("Dictionary size: %d\n", dictionary->Size());
	total_words = 0;
	for (int i = 0; i < dictionary->Size(); ++i)
		total_words += dictionary->GetWordInfo(i)->freq;
	printf("Words in corpus %I64d\n", total_words);
	huffman_encoder->BuildFromTermFrequency(dictionary);
	fclose(fid);
	return total_words;
}
int64_t LoadVocab ( Option * opt , Dictionary * dictionary , HuffmanEncoder * huffman_encoder ) { int64_t total_words = 0 ; char word [ MAX_STRING ] ; FILE * fid = nullptr ; printf ( " \n " , opt -> read_vocab_file ) ; if ( opt -> read_vocab_file != nullptr && strlen ( opt -> read_vocab_file ) > 0 ) { printf ( " \n " , opt -> read_vocab_file ) ; fid = fopen ( opt -> read_vocab_file , " " ) ; int word_freq ; while ( fscanf ( fid , " " , word , & word_freq ) != EOF ) { dictionary -> Insert ( word , word_freq ) ; } } dictionary -> RemoveWordsLessThan ( opt -> min_count ) ; printf ( " \n " , dictionary -> Size ( ) ) ; total_words = 0 ; for ( int i = 0 ; i < dictionary -> Size ( ) ; ++ i ) total_words += dictionary -> GetWordInfo ( i ) -> freq ; printf ( " \n " , total_words ) ; huffman_encoder -> BuildFromTermFrequency ( dictionary ) ; fclose ( fid ) ; return total_words ; }
Read the vocabulary file; create the dictionary and huffman_encoder according opt
Read the vocabulary file ; create the dictionary and huffman_encoder according opt