www/script.js
// converte as palavras em tokens que podem ser usados no TF
converte as palavras em tokens que podem ser usados no TF
converte as palavras em tokens que podem ser usados no TF
function tokenize(wordArray) {
    let returnArray = [DICTIONARY.START];
    for (var i = 0; i < wordArray.length; i++) {
        let encoding = DICTIONARY.LOOKUP[wordArray[i]];
        returnArray.push(encoding === undefined ? DICTIONARY.UNKNOWN : encoding);
    }
    while (i < ENCODING_LENGTH - 1) {
        returnArray.push(DICTIONARY.PAD);
        i++;
    }
    console.log([returnArray]);
    return tf.tensor([returnArray]);
}
function tokenize ( wordArray ) { let returnArray = [ DICTIONARY . START ] ; for ( var i = 0 ; i < wordArray . length ; i ++ ) { let encoding = DICTIONARY . LOOKUP [ wordArray [ i ] ] ; returnArray . push ( encoding === undefined ? DICTIONARY . UNKNOWN : encoding ) ; } while ( i < ENCODING_LENGTH - 1 ) { returnArray . push ( DICTIONARY . PAD ) ; i ++ ; } console . log ( [ returnArray ] ) ; return tf . tensor ( [ returnArray ] ) ; }
converte as palavras em tokens que podem ser usados no TF
converte as palavras em tokens que podem ser usados no TF
// Always start with the START token. // Loop through the words in the sentence you want to encode. // If word is found in dictionary, add that number else // you add the UNKNOWN token. // Finally if the number of words was < the minimum encoding length // minus 1 (due to the start token), fill the rest with PAD tokens. // Log the result to see what you made. // Convert to a TensorFlow Tensor and return that.