deeplearning4j-scaleout/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/loader/WordVectorSerializer.java
/**
     * Read a binary word2vec file.
     *
     * @param modelFile
     *            the File to read
     * @param linebreaks
     *            if true, the reader expects each word/vector to be in a separate line, terminated
     *            by a line break
     * @return a {@link Word2Vec model}
     * @throws NumberFormatException
     * @throws IOException
     * @throws FileNotFoundException
     */
Read a binary word2vec file.
Read a binary word2vec file .
private static Word2Vec readBinaryModel(File modelFile, boolean linebreaks)
            throws NumberFormatException, IOException
    {
        InMemoryLookupTable lookupTable;
        VocabCache cache;
        INDArray syn0;
        int words, size;
        try (BufferedInputStream bis = new BufferedInputStream(
                GzipUtils.isCompressedFilename(modelFile.getName())
                        ? new GZIPInputStream(new FileInputStream(modelFile))
                        : new FileInputStream(modelFile));
             DataInputStream dis = new DataInputStream(bis)) {
            words = Integer.parseInt(readString(dis));
            size = Integer.parseInt(readString(dis));
            syn0 = Nd4j.create(words, size);
            cache = new InMemoryLookupCache(false);
            lookupTable = (InMemoryLookupTable) new InMemoryLookupTable.Builder().cache(cache)
                    .vectorLength(size).build();
            String word;
            for (int i = 0; i < words; i++) {
                word = readString(dis);
                log.trace("Loading " + word + " with word " + i);
                float[] vector = new float[size];
                for (int j = 0; j < size; j++) {
                    vector[j] = readFloat(dis);
                }
                syn0.putRow(i, Transforms.unitVec(Nd4j.create(vector)));
                cache.addWordToIndex(cache.numWords(), word);
                cache.addToken(new VocabWord(1, word));
                cache.putVocabWord(word);
                if (linebreaks) {
                    dis.readByte(); 
                }
            }
        }
        Word2Vec ret = new Word2Vec();
        lookupTable.setSyn0(syn0);
        ret.setVocab(cache);
        ret.setLookupTable(lookupTable);
        return ret;
    }
private static Word2Vec readBinaryModel ( File modelFile , boolean linebreaks ) throws NumberFormatException , IOException { InMemoryLookupTable lookupTable ; VocabCache cache ; INDArray syn0 ; int words , size ; try ( BufferedInputStream bis = new BufferedInputStream ( GzipUtils . isCompressedFilename ( modelFile . getName ( ) ) ? new GZIPInputStream ( new FileInputStream ( modelFile ) ) : new FileInputStream ( modelFile ) ) ; DataInputStream dis = new DataInputStream ( bis ) ) { words = Integer . parseInt ( readString ( dis ) ) ; size = Integer . parseInt ( readString ( dis ) ) ; syn0 = Nd4j . create ( words , size ) ; cache = new InMemoryLookupCache ( false ) ; lookupTable = ( InMemoryLookupTable ) new InMemoryLookupTable . Builder ( ) . cache ( cache ) . vectorLength ( size ) . build ( ) ; String word ; for ( int i = 0 ; i < words ; i ++ ) { word = readString ( dis ) ; log . trace ( "Loading " + word + " with word " + i ) ; float [ ] vector = new float [ size ] ; for ( int j = 0 ; j < size ; j ++ ) { vector [ j ] = readFloat ( dis ) ; } syn0 . putRow ( i , Transforms . unitVec ( Nd4j . create ( vector ) ) ) ; cache . addWordToIndex ( cache . numWords ( ) , word ) ; cache . addToken ( new VocabWord ( 1 , word ) ) ; cache . putVocabWord ( word ) ; if ( linebreaks ) { dis . readByte ( ) ; } } } Word2Vec ret = new Word2Vec ( ) ; lookupTable . setSyn0 ( syn0 ) ; ret . setVocab ( cache ) ; ret . setLookupTable ( lookupTable ) ; return ret ; }
Read a binary word2vec file.
Read a binary word2vec file .
// line break