train/swe_with_2nd_randomwalk.c
// Sorts the vocabulary by frequency using word counts
Sorts the vocabulary by frequency using word counts
Sorts the vocabulary by frequency using word counts
void SortVocab() {
    int a, size;
    unsigned int hash;
    qsort(&vocab[2], vocab_size - 2, sizeof(struct vocab_word), VocabCompare);
    for (a = 0; a < vocab_hash_size; a++) vocab_hash[a] = -1;
    size = vocab_size;
    train_words = 0;
    for (a = 0; a < size; a++) {
        if ((vocab[a].cn < min_count) && (a > 1)) {
            vocab_size--;
            free(vocab[a].word);
        } else {
            hash=GetWordHash(vocab[a].word);
            while (vocab_hash[hash] != -1) hash = (hash + 1) % vocab_hash_size;
            vocab_hash[hash] = a;
            train_words += vocab[a].cn;
        }
    }
    vocab = (struct vocab_word *)realloc(vocab, (vocab_size + 1) * sizeof(struct vocab_word));
    for (a = 0; a < vocab_size; a++) {
        vocab[a].code = (char *)calloc(MAX_CODE_LENGTH, sizeof(char));
        vocab[a].point = (int *)calloc(MAX_CODE_LENGTH, sizeof(int));
    }
}
void SortVocab ( ) { int a , size ; unsigned int hash ; qsort ( & vocab [ 2 ] , vocab_size - 2 , sizeof ( struct vocab_word ) , VocabCompare ) ; for ( a = 0 ; a < vocab_hash_size ; a ++ ) vocab_hash [ a ] = -1 ; size = vocab_size ; train_words = 0 ; for ( a = 0 ; a < size ; a ++ ) { if ( ( vocab [ a ] . cn < min_count ) && ( a > 1 ) ) { vocab_size -- ; free ( vocab [ a ] . word ) ; } else { hash = GetWordHash ( vocab [ a ] . word ) ; while ( vocab_hash [ hash ] != -1 ) hash = ( hash + 1 ) % vocab_hash_size ; vocab_hash [ hash ] = a ; train_words += vocab [ a ] . cn ; } } vocab = ( struct vocab_word * ) realloc ( vocab , ( vocab_size + 1 ) * sizeof ( struct vocab_word ) ) ; for ( a = 0 ; a < vocab_size ; a ++ ) { vocab [ a ] . code = ( char * ) calloc ( MAX_CODE_LENGTH , sizeof ( char ) ) ; vocab [ a ] . point = ( int * ) calloc ( MAX_CODE_LENGTH , sizeof ( int ) ) ; } }
Sorts the vocabulary by frequency using word counts
Sorts the vocabulary by frequency using word counts
// Sort the vocabulary and keep </s> at the first position // Words occuring less than min_count times will be discarded from the vocab // Hash will be re-computed, as after the sorting it is not actual // Allocate memory for the binary tree construction