deeplearning4j-scaleout/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/loader/WordVectorSerializer.java
/**
     * Saves full Word2Vec model in the way, that allows model updates without being rebuilt from scratches
     *
     * @param vec - The Word2Vec instance to be saved
     * @param path - the path for json to be saved
     */
Saves full Word2Vec model in the way, that allows model updates without being rebuilt from scratches
@param vec - The Word2Vec instance to be saved
@param path - the path for json to be saved
Saves full Word2Vec model in the way that allows model updates without being rebuilt from scratches @param vec - The Word2Vec instance to be saved @param path - the path for json to be saved
public static void writeFullModel(@NonNull Word2Vec vec, @NonNull String path) {
        /*
            Basically we need to save:
                    1. WeightLookupTable, especially syn0 and syn1 matrices
                    2. VocabCache, including only WordCounts
                    3. Settings from Word2Vect model: workers, layers, etc.
         */
        PrintWriter printWriter = null;
        try {
            printWriter = new PrintWriter(path);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        WeightLookupTable lookupTable = vec.getLookupTable();
        VocabCache vocabCache = vec.getVocab(); 
        if (!(lookupTable instanceof InMemoryLookupTable)) throw new IllegalStateException("At this moment only InMemoryLookupTable is supported.");
        if (!(vocabCache instanceof InMemoryLookupCache)) throw new IllegalStateException("At this moment only InMemoryLookupCache is supported.");
        VectorsConfiguration conf = vec.getConfiguration();
        conf.setVocabSize(vocabCache.numWords());
        VocabularyHolder holder = new VocabularyHolder.Builder()
                .externalCache(vocabCache)
                .build();
        printWriter.println(conf.toJson());
        log.info("Word2Vec conf. JSON: " + conf.toJson());
        /*
            We have the following map:
            Line 0 - VectorsConfiguration JSON string
            Line 1 - expTable
            Line 2 - table
            All following lines are vocab/weight lookup table saved line by line as VocabularyWord JSON representation
         */
        StringBuilder builder = new StringBuilder();
        for (int x = 0; x < ((InMemoryLookupTable) lookupTable).getExpTable().length; x++) {
            builder.append(((InMemoryLookupTable) lookupTable).getExpTable()[x]).append(" ");
        }
        printWriter.println(builder.toString().trim());
        if (conf.getNegative() > 0) {
            builder = new StringBuilder();
            for (int x = 0; x < ((InMemoryLookupTable) lookupTable).getTable().columns(); x++) {
                builder.append(((InMemoryLookupTable) lookupTable).getTable().getDouble(x)).append(" ");
            }
            printWriter.println(builder.toString().trim());
        } else printWriter.println("");
        List<VocabWord> words = new ArrayList<>(((InMemoryLookupCache) vocabCache).getVocabs().values());
        for (SequenceElement word: words) {
            VocabularyWord vw = new VocabularyWord(word.getLabel());
            vw.setCount(vocabCache.wordFrequency(word.getLabel()));
            vw.setHuffmanNode(VocabularyHolder.buildNode(word.getCodes(), word.getPoints(), word.getCodeLength(), word.getIndex()));
            INDArray syn0 = ((InMemoryLookupTable) lookupTable).getSyn0().getRow(vocabCache.indexOf(word.getLabel()));
            double[] dsyn0 = new double[syn0.columns()];
            for (int x =0; x < conf.getLayersSize(); x++) {
                dsyn0[x] = syn0.getDouble(x);
            }
            vw.setSyn0(dsyn0);
            INDArray syn1 = ((InMemoryLookupTable) lookupTable).getSyn1().getRow(vocabCache.indexOf(word.getLabel()));
            double[] dsyn1 = new double[syn1.columns()];
            for (int x =0; x < syn1.columns(); x++) {
                dsyn1[x] = syn1.getDouble(x);
            }
            vw.setSyn1(dsyn1);
            Neg, if negative sampling is used
            if (conf.getNegative() > 0) {
                INDArray syn1Neg = ((InMemoryLookupTable) lookupTable).getSyn1Neg().getRow(vocabCache.indexOf(word.getLabel()));
                double[] dsyn1Neg = new double[syn1Neg.columns()];
                for (int x = 0; x < syn1Neg.columns(); x++) {
                    dsyn1Neg[x] = syn1Neg.getDouble(x);
                }
                vw.setSyn1Neg(dsyn1Neg);
            }
            if (conf.isUseAdaGrad() && ((InMemoryLookupTable) lookupTable).isUseAdaGrad()) {
                INDArray gradient = word.getHistoricalGradient();
                if (gradient == null) gradient = Nd4j.zeros(word.getCodes().size());
                double ada[] = new double[gradient.columns()];
                for (int x = 0; x < gradient.columns(); x++) {
                    ada[x] = gradient.getDouble(x);
                }
                vw.setHistoricalGradient(ada);
            }
            printWriter.println(vw.toJson());
        }
        printWriter.flush();
        printWriter.close();
    }
public static void writeFullModel ( @ NonNull Word2Vec vec , @ NonNull String path ) { /*
            Basically we need to save:
                    1. WeightLookupTable, especially syn0 and syn1 matrices
                    2. VocabCache, including only WordCounts
                    3. Settings from Word2Vect model: workers, layers, etc.
         */ PrintWriter printWriter = null ; try { printWriter = new PrintWriter ( path ) ; } catch ( Exception e ) { throw new RuntimeException ( e ) ; } WeightLookupTable lookupTable = vec . getLookupTable ( ) ; VocabCache vocabCache = vec . getVocab ( ) ; if ( ! ( lookupTable instanceof InMemoryLookupTable ) ) throw new IllegalStateException ( "At this moment only InMemoryLookupTable is supported." ) ; if ( ! ( vocabCache instanceof InMemoryLookupCache ) ) throw new IllegalStateException ( "At this moment only InMemoryLookupCache is supported." ) ; VectorsConfiguration conf = vec . getConfiguration ( ) ; conf . setVocabSize ( vocabCache . numWords ( ) ) ; VocabularyHolder holder = new VocabularyHolder . Builder ( ) . externalCache ( vocabCache ) . build ( ) ; printWriter . println ( conf . toJson ( ) ) ; log . info ( "Word2Vec conf. JSON: " + conf . toJson ( ) ) ; /*
            We have the following map:
            Line 0 - VectorsConfiguration JSON string
            Line 1 - expTable
            Line 2 - table

            All following lines are vocab/weight lookup table saved line by line as VocabularyWord JSON representation
         */ StringBuilder builder = new StringBuilder ( ) ; for ( int x = 0 ; x < ( ( InMemoryLookupTable ) lookupTable ) . getExpTable ( ) . length ; x ++ ) { builder . append ( ( ( InMemoryLookupTable ) lookupTable ) . getExpTable ( ) [ x ] ) . append ( " " ) ; } printWriter . println ( builder . toString ( ) . trim ( ) ) ; if ( conf . getNegative ( ) > 0 ) { builder = new StringBuilder ( ) ; for ( int x = 0 ; x < ( ( InMemoryLookupTable ) lookupTable ) . getTable ( ) . columns ( ) ; x ++ ) { builder . append ( ( ( InMemoryLookupTable ) lookupTable ) . getTable ( ) . getDouble ( x ) ) . append ( " " ) ; } printWriter . println ( builder . toString ( ) . trim ( ) ) ; } else printWriter . println ( "" ) ; List < VocabWord > words = new ArrayList < > ( ( ( InMemoryLookupCache ) vocabCache ) . getVocabs ( ) . values ( ) ) ; for ( SequenceElement word : words ) { VocabularyWord vw = new VocabularyWord ( word . getLabel ( ) ) ; vw . setCount ( vocabCache . wordFrequency ( word . getLabel ( ) ) ) ; vw . setHuffmanNode ( VocabularyHolder . buildNode ( word . getCodes ( ) , word . getPoints ( ) , word . getCodeLength ( ) , word . getIndex ( ) ) ) ; INDArray syn0 = ( ( InMemoryLookupTable ) lookupTable ) . getSyn0 ( ) . getRow ( vocabCache . indexOf ( word . getLabel ( ) ) ) ; double [ ] dsyn0 = new double [ syn0 . columns ( ) ] ; for ( int x = 0 ; x < conf . getLayersSize ( ) ; x ++ ) { dsyn0 [ x ] = syn0 . getDouble ( x ) ; } vw . setSyn0 ( dsyn0 ) ; INDArray syn1 = ( ( InMemoryLookupTable ) lookupTable ) . getSyn1 ( ) . getRow ( vocabCache . indexOf ( word . getLabel ( ) ) ) ; double [ ] dsyn1 = new double [ syn1 . columns ( ) ] ; for ( int x = 0 ; x < syn1 . columns ( ) ; x ++ ) { dsyn1 [ x ] = syn1 . getDouble ( x ) ; } vw . setSyn1 ( dsyn1 ) ; if ( conf . getNegative ( ) > 0 ) { INDArray syn1Neg = ( ( InMemoryLookupTable ) lookupTable ) . getSyn1Neg ( ) . getRow ( vocabCache . indexOf ( word . getLabel ( ) ) ) ; double [ ] dsyn1Neg = new double [ syn1Neg . columns ( ) ] ; for ( int x = 0 ; x < syn1Neg . columns ( ) ; x ++ ) { dsyn1Neg [ x ] = syn1Neg . getDouble ( x ) ; } vw . setSyn1Neg ( dsyn1Neg ) ; } if ( conf . isUseAdaGrad ( ) && ( ( InMemoryLookupTable ) lookupTable ) . isUseAdaGrad ( ) ) { INDArray gradient = word . getHistoricalGradient ( ) ; if ( gradient == null ) gradient = Nd4j . zeros ( word . getCodes ( ) . size ( ) ) ; double ada [ ] = new double [ gradient . columns ( ) ] ; for ( int x = 0 ; x < gradient . columns ( ) ; x ++ ) { ada [ x ] = gradient . getDouble ( x ) ; } vw . setHistoricalGradient ( ada ) ; } printWriter . println ( vw . toJson ( ) ) ; } printWriter . flush ( ) ; printWriter . close ( ) ; }
Saves full Word2Vec model in the way, that allows model updates without being rebuilt from scratches
 @param vec - The Word2Vec instance to be saved
 @param path - the path for json to be saved
Saves full Word2Vec model in the way that allows model updates without being rebuilt from scratches @param vec - The Word2Vec instance to be saved @param path - the path for json to be saved
// ((InMemoryLookupTable) lookupTable).getVocab(); //vec.getVocab(); // actually we don't need expTable, since it produces exact results on subsequent runs untill you dont modify expTable size :) // saving ExpTable just for "special case in future" // saving table, available only if negative sampling is used // writing down syn0 // writing down syn1 // writing down syn1Neg, if negative sampling is used // in case of UseAdaGrad == true - we should save gradients for each word in vocab // at this moment we have whole vocab serialized