deeplearning4j-scaleout/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/loader/WordVectorSerializer.java
/**
     * Loads an in memory cache from the given path (sets syn0 and the vocab)
     *
     * @param vectorsFile the path of the file to load
     * @return a Pair holding the lookup table and the vocab cache.
     * @throws FileNotFoundException if the input file does not exist
     */
Loads an in memory cache from the given path (sets syn0 and the vocab)
@param vectorsFile the path of the file to load
@return a Pair holding the lookup table and the vocab cache.
@throws FileNotFoundException if the input file does not exist
Loads an in memory cache from the given path ( sets syn0 and the vocab ) @param vectorsFile the path of the file to load @return a Pair holding the lookup table and the vocab cache . @throws FileNotFoundException if the input file does not exist
public static Pair<InMemoryLookupTable, VocabCache> loadTxt(File vectorsFile)
            throws FileNotFoundException {
        BufferedReader reader = new BufferedReader(new FileReader(vectorsFile));
        VocabCache cache = new InMemoryLookupCache();
        LineIterator iter = IOUtils.lineIterator(reader);
        String line = null;
        boolean hasHeader = false;
        if (iter.hasNext()) {
            line = iter.nextLine();    
            if(!line.contains(" "))
                hasHeader = true;
        }
        if(hasHeader) {
            iter.close();
            iter = IOUtils.lineIterator(reader);
            iter.nextLine();
        }
        List<INDArray> arrays = new ArrayList<>();
        while (iter.hasNext()) {
            line = iter.nextLine();
            String[] split = line.split(" ");
            String word = split[0];
            VocabWord word1 = new VocabWord(1.0, word);
            cache.addToken(word1);
            cache.addWordToIndex(cache.numWords(), word);
            word1.setIndex(cache.numWords());
            cache.putVocabWord(word);
            INDArray row = Nd4j.create(Nd4j.createBuffer(split.length - 1));
            for (int i = 1; i < split.length; i++) {
                row.putScalar(i - 1, Float.parseFloat(split[i]));
            }
            arrays.add(row);
        }
        INDArray syn = Nd4j.create(new int[]{arrays.size(), arrays.get(0).columns()});
        for (int i = 0; i < syn.rows(); i++) {
            syn.putRow(i, arrays.get(i));
        }
        InMemoryLookupTable lookupTable = (InMemoryLookupTable) new InMemoryLookupTable.Builder()
                .vectorLength(arrays.get(0).columns())
                .useAdaGrad(false).cache(cache)
                .build();
        Nd4j.clearNans(syn);
        lookupTable.setSyn0(syn);
        iter.close();
        return new Pair<>(lookupTable, cache);
    }
public static Pair < InMemoryLookupTable , VocabCache > loadTxt ( File vectorsFile ) throws FileNotFoundException { BufferedReader reader = new BufferedReader ( new FileReader ( vectorsFile ) ) ; VocabCache cache = new InMemoryLookupCache ( ) ; LineIterator iter = IOUtils . lineIterator ( reader ) ; String line = null ; boolean hasHeader = false ; if ( iter . hasNext ( ) ) { line = iter . nextLine ( ) ; if ( ! line . contains ( " " ) ) hasHeader = true ; } if ( hasHeader ) { iter . close ( ) ; iter = IOUtils . lineIterator ( reader ) ; iter . nextLine ( ) ; } List < INDArray > arrays = new ArrayList < > ( ) ; while ( iter . hasNext ( ) ) { line = iter . nextLine ( ) ; String [ ] split = line . split ( " " ) ; String word = split [ 0 ] ; VocabWord word1 = new VocabWord ( 1.0 , word ) ; cache . addToken ( word1 ) ; cache . addWordToIndex ( cache . numWords ( ) , word ) ; word1 . setIndex ( cache . numWords ( ) ) ; cache . putVocabWord ( word ) ; INDArray row = Nd4j . create ( Nd4j . createBuffer ( split . length - 1 ) ) ; for ( int i = 1 ; i < split . length ; i ++ ) { row . putScalar ( i - 1 , Float . parseFloat ( split [ i ] ) ) ; } arrays . add ( row ) ; } INDArray syn = Nd4j . create ( new int [ ] { arrays . size ( ) , arrays . get ( 0 ) . columns ( ) } ) ; for ( int i = 0 ; i < syn . rows ( ) ; i ++ ) { syn . putRow ( i , arrays . get ( i ) ) ; } InMemoryLookupTable lookupTable = ( InMemoryLookupTable ) new InMemoryLookupTable . Builder ( ) . vectorLength ( arrays . get ( 0 ) . columns ( ) ) . useAdaGrad ( false ) . cache ( cache ) . build ( ) ; Nd4j . clearNans ( syn ) ; lookupTable . setSyn0 ( syn ) ; iter . close ( ) ; return new Pair < > ( lookupTable , cache ) ; }
Loads an in memory cache from the given path (sets syn0 and the vocab)
 @param vectorsFile the path of the file to load
 @return a Pair holding the lookup table and the vocab cache.
Loads an in memory cache from the given path ( sets syn0 and the vocab ) @param vectorsFile the path of the file to load @return a Pair holding the lookup table and the vocab cache .
// skip header line //look for spaces //reposition buffer to be one line ahead