train/swe_with_2nd_randomwalk.c
// Reduces the vocabulary by removing infrequent tokens
Reduces the vocabulary by removing infrequent tokens
Reduces the vocabulary by removing infrequent tokens
void ReduceVocab() {
    int a, b = 2;
    unsigned int hash;
    for (a = 2; a < vocab_size; a++) if (vocab[a].cn > min_reduce) {
        vocab[b].cn = vocab[a].cn;
        vocab[b].word = vocab[a].word;
        b++;
    } else free(vocab[a].word);
    vocab_size = b;
    for (a = 2; a < vocab_hash_size; a++) vocab_hash[a] = -1;
    for (a = 2; a < vocab_size; a++) {
        hash = GetWordHash(vocab[a].word);
        while (vocab_hash[hash] != -1) hash = (hash + 1) % vocab_hash_size;
        vocab_hash[hash] = a;
    }
    fflush(stdout);
    min_reduce++;
}
void ReduceVocab ( ) { int a , b = 2 ; unsigned int hash ; for ( a = 2 ; a < vocab_size ; a ++ ) if ( vocab [ a ] . cn > min_reduce ) { vocab [ b ] . cn = vocab [ a ] . cn ; vocab [ b ] . word = vocab [ a ] . word ; b ++ ; } else free ( vocab [ a ] . word ) ; vocab_size = b ; for ( a = 2 ; a < vocab_hash_size ; a ++ ) vocab_hash [ a ] = -1 ; for ( a = 2 ; a < vocab_size ; a ++ ) { hash = GetWordHash ( vocab [ a ] . word ) ; while ( vocab_hash [ hash ] != -1 ) hash = ( hash + 1 ) % vocab_hash_size ; vocab_hash [ hash ] = a ; } fflush ( stdout ) ; min_reduce ++ ; }
Reduces the vocabulary by removing infrequent tokens
Reduces the vocabulary by removing infrequent tokens
// Hash will be re-computed, as it is not actual